Update README (#9)
* Add black tool config * Add missing backend env files * Updated with development docs * Lint
Graham Williamson committed
Jul 13, 2023 at 00:52 UTC
2b846404ff50f7373eab515ff7ec1872aa1d662e
25 files changed
+139
-140
.flake8
+2
-1
@@ -3,7 +3,8 @@
3
max-line-length = 180
4
#select = B,C,E,F,W,T4,B9
5
#ignore = E203, E266, E501, W503, F403, F401
6
-ignore = W503, E231, W605
6
+ignore = E402, W503, E231, W605
7
+ # E402, # module level import not at top of file (using isort)
8
# W503, # line break before binary operator
9
# E231, # missing whitespace after ',' (caused by black style)
10
# W605, # invalid escape sequence (caused by regex)
.isort.cfg
deleted
-4
@@ -1,4 +0,0 @@
1
-[settings]
2
-profile=black
3
-force_single_line=True
4
-src_paths=backend
.pre-commit-config.yaml
+3
-3
@@ -22,18 +22,18 @@ repos:
22
name: Sort python imports (fixes files)
23
24
- repo: https://github.com/psf/black
25
- rev: 23.3.0
25
+ rev: 23.7.0
26
hooks:
27
- id: black
28
language_version: python3.11
29
30
- repo: https://github.com/asottile/setup-cfg-fmt
31
- rev: v2.3.0
31
+ rev: v2.4.0
32
hooks:
33
- id: setup-cfg-fmt
34
35
- repo: https://github.com/asottile/add-trailing-comma
36
- rev: v2.4.0
36
+ rev: v3.0.0
37
hooks:
38
- id: add-trailing-comma
39
README.md
+50
@@ -1,3 +1,53 @@
1
# CoPilot
2
3
SOCFortress CoPilot
4
+
5
+# Development
6
+
7
+## Local development of backend
8
+
9
+Setup the env vars, adjust if required.
10
+
11
+```
12
+cd backend
13
+cp .env.example .env
14
+```
15
+
16
+Create and activate python, installing dependencies
17
+
18
+```
19
+python3.11 -m venv.venv --copies
20
+source .venv/bin/activate
21
+pip install -U pip setuptools wheel
22
+pip install -r requirements.in
23
+```
24
+
25
+Create a DB and apply any pending DB migrations
26
+
27
+```
28
+FLASK_APP=copilot.py flask db upgrade
29
+```
30
+
31
+Start local dev server
32
+
33
+```
34
+python3 app.py
35
+```
36
+
37
+If there any changes made to the model run the migrate command (example commment)
38
+and if any changes were detected, update your local DB instance.
39
+
40
+```
41
+FLASK_APP=copilot.py flask db migrate -m "Add User model."
42
+FLASK_APP=copilot.py flask db upgrade
43
+```
44
+
45
+See https://flask-migrate.readthedocs.io/en/latest/ for further information
46
+
47
+# Deployment
48
+
49
+## Production Deployment Notes
50
+
51
+```
52
+pip install -r requirements.txt
53
+```
backend/.env.example
new
+5
@@ -0,0 +1,5 @@
1
+DEBUG=True
2
+SECRET_KEY="not so secret"
3
+ENV=development
4
+SQLALCHEMY_TRACK_MODIFICATIONS=True
5
+UPLOAD_FOLDER="/tmp/copilot_uploads"
backend/.env.prod.example
new
+6
@@ -0,0 +1,6 @@
1
+DEBUG=False
2
+SECRET_KEY="not so secret"
3
+ENV=production
4
+SQLALCHEMY_DATABASE_URI="postgresql://postgres:root@localhost:5432/copilot"
5
+SQLALCHEMY_TRACK_MODIFICATIONS=False
6
+UPLOAD_FOLDER="/opt/socfortress/copilot/uploads"
backend/app/__init__.py
+10
-14
@@ -5,8 +5,6 @@ from flask_migrate import Migrate
5
from flask_sqlalchemy import SQLAlchemy
6
from flask_swagger_ui import get_swaggerui_blueprint
7
8
-# from app.routes import bp # Import the blueprint
9
-
8
app = Flask(__name__)
9
10
SWAGGER_URL = "/api/docs" # URL for exposing Swagger UI (without trailing '/')
@@ -27,25 +25,23 @@ swaggerui_blueprint = get_swaggerui_blueprint(
25
)
26
27
app.register_blueprint(swaggerui_blueprint)
30
-
28
CORS(app)
32
-
33
-
29
app.config.from_object("settings")
30
31
db = SQLAlchemy(app)
32
migrate = Migrate(app, db)
33
ma = Marshmallow(app)
34
40
-from app.routes.agents import bp as agents_bp # Import the blueprint
41
-from app.routes.alerts import bp as alerts_bp # Import the blueprint
42
-from app.routes.connectors import bp as connectors_bp # Import the blueprint
43
-from app.routes.dfir_iris import bp as dfir_iris_bp # Import the blueprint
44
-from app.routes.graylog import bp as graylog_bp # Import the blueprint
45
-from app.routes.rules import bp as rules_bp # Import the blueprint
46
-from app.routes.shuffle import bp as shuffle_bp # Import the blueprint
47
-from app.routes.velociraptor import bp as velociraptor_bp # Import the blueprint
48
-from app.routes.wazuhindexer import bp as wazuhindexer_bp # Import the blueprint
35
+
36
+from app.routes.agents import bp as agents_bp
37
+from app.routes.alerts import bp as alerts_bp
38
+from app.routes.connectors import bp as connectors_bp
39
+from app.routes.dfir_iris import bp as dfir_iris_bp
40
+from app.routes.graylog import bp as graylog_bp
41
+from app.routes.rules import bp as rules_bp
42
+from app.routes.shuffle import bp as shuffle_bp
43
+from app.routes.velociraptor import bp as velociraptor_bp
44
+from app.routes.wazuhindexer import bp as wazuhindexer_bp
45
46
app.register_blueprint(connectors_bp) # Register the connectors blueprint
47
app.register_blueprint(agents_bp) # Register the agents blueprint
backend/app/models/connectors.py
+2
-10
@@ -66,17 +66,9 @@ class Connector(ABC):
66
Raises:
67
NoResultFound: If the connector_name is not found in the database.
68
"""
69
- connector = (
70
- current_app.extensions["sqlalchemy"]
71
- .db.session.query(Connectors)
72
- .filter_by(connector_name=connector_name)
73
- .first()
74
- )
69
+ connector = current_app.extensions["sqlalchemy"].db.session.query(Connectors).filter_by(connector_name=connector_name).first()
70
if connector:
76
- attributes = {
77
- col.name: getattr(connector, col.name)
78
- for col in Connectors.__table__.columns
79
- }
71
+ attributes = {col.name: getattr(connector, col.name) for col in Connectors.__table__.columns}
72
return attributes
73
else:
74
raise NoResultFound
backend/app/models/graylog.py
+2
-6
@@ -88,9 +88,5 @@ class GraylogMetricsAllocationSchema(ma.Schema):
88
)
89
90
91
-graylog_metrics_allocation_schema: GraylogMetricsAllocationSchema = (
92
- GraylogMetricsAllocationSchema()
93
-)
94
-graylog_metrics_allocations_schema: GraylogMetricsAllocationSchema = (
95
- GraylogMetricsAllocationSchema(many=True)
96
-)
91
+graylog_metrics_allocation_schema: GraylogMetricsAllocationSchema = GraylogMetricsAllocationSchema()
92
+graylog_metrics_allocations_schema: GraylogMetricsAllocationSchema = GraylogMetricsAllocationSchema(many=True)
backend/app/models/models.py
+1
-5
@@ -128,11 +128,7 @@ class Connectors(db.Model):
128
self.connector_username = connector_username
129
self.connector_password = connector_password
130
131
- if (
132
- connector_name.lower() == "shuffle"
133
- or connector_name.lower() == "dfir-irs"
134
- or connector_name.lower() == "velociraptor"
135
- ):
131
+ if connector_name.lower() == "shuffle" or connector_name.lower() == "dfir-irs" or connector_name.lower() == "velociraptor":
132
logger.info(f"Setting the API key for {connector_name}")
133
self.connector_api_key = connector_api_key
134
else:
backend/app/models/wazuh_indexer.py
+2
-6
@@ -87,9 +87,5 @@ class WazuhIndexerAllocationSchema(ma.Schema):
87
)
88
89
90
-wazuh_indexer_allocation_schema: WazuhIndexerAllocationSchema = (
91
- WazuhIndexerAllocationSchema()
92
-)
93
-wazuh_indexer_allocations_schema: WazuhIndexerAllocationSchema = (
94
- WazuhIndexerAllocationSchema(many=True)
95
-)
90
+wazuh_indexer_allocation_schema: WazuhIndexerAllocationSchema = WazuhIndexerAllocationSchema()
91
+wazuh_indexer_allocations_schema: WazuhIndexerAllocationSchema = WazuhIndexerAllocationSchema(many=True)
backend/app/routes/connectors.py
+3
@@ -20,6 +20,7 @@ def list_connectors_available():
20
Returns:
21
json: A JSON response containing the list of all available connectors along with their connection verification status.
22
"""
23
+ logger.info("Received request to get all available connectors")
24
connectors_service = ConnectorService(db)
25
connectors = ConnectorsAvailable.query.all()
26
result = connectors_available_schema.dump(connectors)
@@ -44,6 +45,7 @@ def get_connector_details(id: str):
45
Returns:
46
json: A JSON response containing the details of the connector.
47
"""
48
+ logger.info("Received request to get a connector details")
49
service = ConnectorService(db)
50
connector = service.validate_connector_exists(int(id))
51
@@ -67,6 +69,7 @@ def update_connector_route(id: str):
69
json: A JSON response containing the success status of the update operation and a message indicating the status.
70
If the update operation was successful, it returns the connector name and the status of the connection verification.
71
"""
72
+ logger.info("Received request to update connector")
73
api_key_connector = ["Shuffle", "DFIR-IRIS", "Velociraptor"]
74
75
request_data = request.get_json()
backend/app/routes/graylog.py
+6
-3
@@ -1,14 +1,12 @@
1
from flask import Blueprint
2
from flask import jsonify
3
+from loguru import logger
4
5
from app.services.Graylog.index import IndexService
6
from app.services.Graylog.inputs import InputsService
7
from app.services.Graylog.messages import MessagesService
8
from app.services.Graylog.metrics import MetricsService
9
9
-# from loguru import logger
10
-
11
-
10
bp = Blueprint("graylog", __name__)
11
12
@@ -20,6 +18,7 @@ def get_messages() -> dict:
18
Returns:
19
dict: A JSON object containing the list of all the messages.
20
"""
21
+ logger.info("Received request to get graylog messages")
22
service = MessagesService()
23
messages = service.collect_messages()
24
return messages
@@ -34,6 +33,7 @@ def get_metrics() -> dict:
33
dict: A JSON object containing the list of all metrics,
34
including the uncommitted journal size.
35
"""
36
+ logger.info("Received request to get graylog metrics")
37
service = MetricsService()
38
uncommitted_journal_size = service.collect_uncommitted_journal_size()
39
metrics = service.collect_throughput_metrics()
@@ -50,6 +50,7 @@ def get_indices() -> dict:
50
Returns:
51
dict: A JSON object containing the list of all indices.
52
"""
53
+ logger.info("Received request to get graylog indexes")
54
service = IndexService()
55
indices = service.collect_indices()
56
return indices
@@ -66,6 +67,7 @@ def delete_index(index_name: str) -> dict:
67
Returns:
68
dict: A JSON object containing the result of the deletion operation.
69
"""
70
+ logger.info("Received request to delete index")
71
service = IndexService()
72
result = service.delete_index(index_name)
73
return result
@@ -79,6 +81,7 @@ def get_inputs() -> dict:
81
Returns:
82
dict: A JSON object containing the list of all running and configured inputs.
83
"""
84
+ logger.info("Received request to get graylog inputs")
85
service = InputsService()
86
running_inputs = service.collect_running_inputs()
87
configured_inputs = service.collect_configured_inputs()
backend/app/routes/shuffle.py
+4
-8
@@ -1,16 +1,9 @@
1
from flask import Blueprint
2
from flask import jsonify
3
+from loguru import logger
4
4
-# from app.models.connectors import Connector
5
-# from app.models.connectors import WazuhManagerConnector
6
-# from app.services.agents.agents import AgentService
7
-# from app.services.agents.agents import AgentSyncService
5
from app.services.Shuffle.workflows import WorkflowsService
6
10
-# from flask import request
11
-# from loguru import logger
12
-
13
-
7
bp = Blueprint("shuffle", __name__)
8
9
@@ -22,6 +15,7 @@ def get_workflows() -> jsonify:
15
Returns:
16
jsonify: A JSON response containing the list of all configured Workflows in Shuffle.
17
"""
18
+ logger.info("Received request to get all Shuffle workflows")
19
service = WorkflowsService()
20
workflows = service.collect_workflows()
21
return workflows
@@ -37,6 +31,7 @@ def get_workflows_executions() -> jsonify:
31
Returns:
32
jsonify: A JSON response containing the list of all configured workflows and their last execution status.
33
"""
34
+ logger.info("Received request to get all Shuffle workflow execution status")
35
service = WorkflowsService()
36
workflow_details = service.collect_workflow_details()
37
if "workflows" not in workflow_details:
@@ -62,6 +57,7 @@ def get_workflow_executions(workflow_id: str) -> jsonify:
57
Returns:
58
jsonify: A JSON response containing the last execution status of the specified workflow.
59
"""
60
+ logger.info("Received request to get a Shuffle workflow")
61
service = WorkflowsService()
62
workflow_details = service.collect_workflow_executions_status(workflow_id)
63
return workflow_details
backend/app/routes/velociraptor.py
+1
-3
@@ -84,9 +84,7 @@ def collect_artifact():
84
artifact_name = req_data["artifact_name"]
85
client_name = req_data["client_name"]
86
service = UniversalService()
87
- client_id = service.get_client_id(client_name=client_name)["results"][0][
88
- "client_id"
89
- ]
87
+ client_id = service.get_client_id(client_name=client_name)["results"][0]["client_id"]
88
if client_id is None:
89
return (
90
jsonify(
backend/app/services/DFIR_IRIS/universal.py
+1
-3
@@ -90,9 +90,7 @@ class UniversalService:
90
"message": "Connection to DFIR-IRIS unsuccessful.",
91
}
92
93
- def fetch_and_parse_data(
94
- self, session: ClientSession, action: Callable, *args
95
- ) -> Dict[str, Union[bool, Optional[Dict]]]:
93
+ def fetch_and_parse_data(self, session: ClientSession, action: Callable, *args) -> Dict[str, Union[bool, Optional[Dict]]]:
94
"""
95
Fetches and parses data from DFIR-IRIS using a specified action.
96
backend/app/services/Graylog/index.py
+3
-12
@@ -33,11 +33,7 @@ class IndexService:
33
Returns:
34
dict: A dictionary containing the success status, a message, and potentially a dictionary with indices.
35
"""
36
- if (
37
- self.connector_url is None
38
- or self.connector_username is None
39
- or self.connector_password is None
40
- ):
36
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
37
return {"message": "Failed to collect Graylog details", "success": False}
38
39
managed_indices = self._collect_managed_indices()
@@ -95,11 +91,7 @@ class IndexService:
91
dict: A dictionary containing the response.
92
"""
93
logger.info(f"Deleting index {index_name} from Graylog")
98
- if (
99
- self.connector_url is None
100
- or self.connector_username is None
101
- or self.connector_password is None
102
- ):
94
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
95
return {"message": "Failed to collect Graylog details", "success": False}
96
97
# Check if the index exists in Graylog
@@ -143,7 +135,6 @@ class IndexService:
135
except Exception as e:
136
logger.error(f"Failed to delete index {index_name} from Graylog: {e}")
137
return {
146
- "message": f"Failed to delete index {index_name} from Graylog. If this is the current index, "
147
- "it cannot be deleted.",
138
+ "message": f"Failed to delete index {index_name} from Graylog. If this is the current index, " "it cannot be deleted.",
139
"success": False,
140
}
backend/app/services/Graylog/inputs.py
+2
-10
@@ -37,11 +37,7 @@ class InputsService:
37
Returns:
38
dict: A dictionary containing the success status, a message, and potentially a list of running inputs.
39
"""
40
- if (
41
- self.connector_url is None
42
- or self.connector_username is None
43
- or self.connector_password is None
44
- ):
40
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
41
return {"message": "Failed to collect Graylog details", "success": False}
42
43
running_inputs = self._collect_running_inputs()
@@ -92,11 +88,7 @@ class InputsService:
88
Returns:
89
dict: A dictionary containing the success status, a message, and potentially a list of configured inputs.
90
"""
95
- if (
96
- self.connector_url is None
97
- or self.connector_username is None
98
- or self.connector_password is None
99
- ):
91
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
92
return {"message": "Failed to collect Graylog details", "success": False}
93
94
configured_inputs = self._collect_configured_inputs()
backend/app/services/Graylog/messages.py
+1
-5
@@ -48,11 +48,7 @@ class MessagesService:
48
Returns:
49
dict: A dictionary containing the success status, a message, and potentially a list of Graylog messages.
50
"""
51
- if (
52
- self.connector_url is None
53
- or self.connector_username is None
54
- or self.connector_password is None
55
- ):
51
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
52
return {"message": "Failed to collect Graylog details", "success": False}
53
else:
54
try:
backend/app/services/Graylog/metrics.py
+2
-10
@@ -46,11 +46,7 @@ class MetricsService:
46
Returns:
47
dict: A dictionary containing the success status, a message, and the size of uncommitted journal entries.
48
"""
49
- if (
50
- self.connector_url is None
51
- or self.connector_username is None
52
- or self.connector_password is None
53
- ):
49
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
50
return {"message": "Failed to collect Graylog details", "success": False}
51
52
journal_size = self._collect_metrics_uncommitted_journal_size(
@@ -79,11 +75,7 @@ class MetricsService:
75
Returns:
76
dict: A dictionary containing the success status, a message, and the list of throughput metrics.
77
"""
82
- if (
83
- self.connector_url is None
84
- or self.connector_username is None
85
- or self.connector_password is None
86
- ):
78
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
79
return {"message": "Failed to collect Graylog details", "success": False}
80
81
throughput_usage = self._collect_metrics_throughput_usage(
backend/app/services/Velociraptor/artifacts.py
+1
-5
@@ -60,11 +60,7 @@ class ArtifactsService:
60
if not artifacts_response["success"]:
61
return artifacts_response
62
63
- filtered_artifacts = [
64
- artifact
65
- for artifact in artifacts_response["results"]
66
- if artifact["name"].startswith(prefix)
67
- ]
63
+ filtered_artifacts = [artifact for artifact in artifacts_response["results"] if artifact["name"].startswith(prefix)]
64
65
return {
66
"success": True,
backend/app/services/Velociraptor/universal.py
+3
-9
@@ -172,12 +172,8 @@ class UniversalService:
172
"""
173
# Formulate queries
174
try:
175
- vql_client_id = (
176
- f"select client_id from clients(search='host:{client_name}')"
177
- )
178
- vql_last_seen_at = (
179
- f"select last_seen_at from clients(search='host:{client_name}')"
180
- )
175
+ vql_client_id = f"select client_id from clients(search='host:{client_name}')"
176
+ vql_last_seen_at = f"select last_seen_at from clients(search='host:{client_name}')"
177
178
# Get the last seen timestamp
179
last_seen_at = self._get_last_seen_timestamp(vql_last_seen_at)
@@ -221,6 +217,4 @@ class UniversalService:
217
Returns:
218
bool: True if the client is offline, False otherwise.
219
"""
224
- return (
225
- datetime.now() - datetime.fromtimestamp(last_seen_at / 1000000)
226
- ).total_seconds() > 30
220
+ return (datetime.now() - datetime.fromtimestamp(last_seen_at / 1000000)).total_seconds() > 30
backend/app/services/WazuhIndexer/universal.py
+1
-5
@@ -67,11 +67,7 @@ class UniversalService:
67
Returns:
68
list: A list containing the indices.
69
"""
70
- if (
71
- self.connector_url is None
72
- or self.connector_username is None
73
- or self.connector_password is None
74
- ):
70
+ if self.connector_url is None or self.connector_username is None or self.connector_password is None:
71
return {
72
"message": "Failed to collect Wazuh-Indexer details",
73
"success": False,
backend/app/services/connectors/connectors.py
+6
-18
@@ -1,8 +1,8 @@
1
+# from app.models.connectors import ConnectorFactory
2
from flask import current_app
3
from loguru import logger
4
from sqlalchemy.exc import SQLAlchemyError
5
5
-# from app.models.connectors import ConnectorFactory
6
from app.models.connectors import Connector
7
from app.models.connectors import connector_factory
8
from app.models.models import Connectors
@@ -42,9 +42,7 @@ class ConnectorService:
42
"""
43
logger.info(f"Updating connector {connector_id} with data {updated_data}")
44
try:
45
- connector = (
46
- self.db.session.query(Connectors).filter_by(id=connector_id).first()
47
- )
45
+ connector = self.db.session.query(Connectors).filter_by(id=connector_id).first()
46
if connector:
47
for key, value in updated_data.items():
48
if hasattr(connector, key):
@@ -95,12 +93,7 @@ class ConnectorService:
93
If a database error occurred, it returns the error message.
94
"""
95
try:
98
- connector = (
99
- current_app.extensions["sqlalchemy"]
100
- .db.session.query(Connectors)
101
- .filter_by(id=connector_id)
102
- .first()
103
- )
96
+ connector = current_app.extensions["sqlalchemy"].db.session.query(Connectors).filter_by(id=connector_id).first()
97
if connector:
98
return {
99
"message": "Connector exists",
@@ -130,9 +123,7 @@ class ConnectorService:
123
If a database error occurred, it returns the error message.
124
"""
125
try:
133
- connector = (
134
- self.db.session.query(Connectors).filter_by(id=connector_id).first()
135
- )
126
+ connector = self.db.session.query(Connectors).filter_by(id=connector_id).first()
127
if connector is None:
128
return {
129
"message": f"No connector found with id {connector_id}",
@@ -168,9 +159,7 @@ class ConnectorService:
159
If a database error occurred, it returns the error message.
160
"""
161
try:
171
- connector = (
172
- self.db.session.query(Connectors).filter_by(id=connector_id).first()
173
- )
162
+ connector = self.db.session.query(Connectors).filter_by(id=connector_id).first()
163
if connector is None:
164
return {
165
"message": f"No connector found with id {connector_id}",
@@ -218,8 +207,7 @@ class ConnectorService:
207
return {"message": "Request data is valid", "success": True}
208
else:
209
return {
221
- "message": "Request data is invalid. Ensure connector_url, connector_username and connector_password "
222
- "are present",
210
+ "message": "Request data is invalid. Ensure connector_url, connector_username and connector_password " "are present",
211
"success": False,
212
}
213
pyproject.toml
new
+22
@@ -0,0 +1,22 @@
1
+[tool.black]
2
+line-length = 140
3
+target-version = ['py38', 'py39']
4
+include = '\.pyi?$'
5
+exclude = '''
6
+/(
7
+ \.git
8
+ | \.hg
9
+ | \.mypy_cache
10
+ | \.tox
11
+ | \.venv
12
+ | _build
13
+ | buck-out
14
+ | build
15
+ | dist
16
+)/
17
+'''
18
+
19
+[tool.isort]
20
+profile = "black"
21
+force_single_line = true
22
+src_paths = "backend"