+
+import orjson
+from xml.sax.saxutils import escape as xml_escape
+from datetime import date
+
+from fastapi import APIRouter, Response, Depends
+from fastapi.responses import ORJSONResponse
+from fastapi.exceptions import RequestValidationError
+
+import anybadge
+
+from api.object_specifications import Software
+from api.api_helpers import (ORJSONResponseObjKeep, add_phase_stats_statistics,
+ determine_comparison_case,get_comparison_details,
+ html_escape_multi, get_phase_stats, get_phase_stats_object,
+ is_valid_uuid, convert_value, get_timeline_query,
+ get_run_info, get_machine_list, get_artifact, store_artifact,
+ authenticate, check_int_field_api)
+
+from lib.global_config import GlobalConfig
+from lib.db import DB
+from lib.diff import get_diffable_rows, diff_rows
+from lib.job.base import Job
+from lib.user import User
+from lib.timeline_project import TimelineProject
+from lib import utils
+
+from enum import Enum
+ArtifactType = Enum('ArtifactType', ['DIFF', 'COMPARE', 'STATS', 'BADGE'])
+
+
+router = APIRouter()
+
+
+# Return a list of all known machines in the cluster
+@router.get('/v1/machines')
+async def get_machines(
+ user: User = Depends(authenticate), # pylint: disable=unused-argument
+ ):
+
+ data = get_machine_list()
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data})
+
+@router.get('/v1/jobs')
+async def get_jobs(
+ machine_id: int | None = None,
+ state: str | None = None,
+ user: User = Depends(authenticate), # pylint: disable=unused-argument
+ ):
+
+ params = []
+ machine_id_condition = ''
+ state_condition = ''
+
+ if machine_id and check_int_field_api(machine_id, 'machine_id', 1024):
+ machine_id_condition = 'AND j.machine_id = %s'
+ params.append(machine_id)
+
+ if state is not None and state != '':
+ state_condition = 'AND j.state = %s'
+ params.append(state)
+
+ query = f"""
+ SELECT j.id, r.id as run_id, j.name, j.url, j.filename, j.branch, m.description, j.state, j.updated_at, j.created_at
+ FROM jobs as j
+ LEFT JOIN machines as m on m.id = j.machine_id
+ LEFT JOIN runs as r on r.job_id = j.id
+ WHERE
+ j.type = 'run'
+ {machine_id_condition}
+ {state_condition}
+ ORDER BY j.updated_at DESC, j.created_at ASC
+ """
+ data = DB().fetch_all(query, params)
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data})
+
+# A route to return all of the available entries in our catalog.
+@router.get('/v1/notes/{run_id}')
+async def get_notes(run_id, user: User = Depends(authenticate)):
+ if run_id is None or not is_valid_uuid(run_id):
+ raise RequestValidationError('Run ID is not a valid UUID or empty')
+
+ query = '''
+ SELECT n.run_id, n.detail_name, n.note, n.time
+ FROM notes as n
+ JOIN runs as r on n.run_id = r.id
+ WHERE
+ (TRUE = %s OR r.user_id = ANY(%s::int[]))
+ AND n.run_id = %s
+ ORDER BY n.created_at DESC -- important to order here, the charting library in JS cannot do that automatically!
+ '''
+
+ params = (user.is_super_user(), user.visible_users(), run_id)
+ data = DB().fetch_all(query, params=params)
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ escaped_data = [html_escape_multi(note) for note in data]
+ return ORJSONResponseObjKeep({'success': True, 'data': escaped_data})
+
+
+@router.get('/v1/network/{run_id}')
+async def get_network(run_id, user: User = Depends(authenticate)):
+ if run_id is None or not is_valid_uuid(run_id):
+ raise RequestValidationError('Run ID is not a valid UUID or empty')
+
+ query = '''
+ SELECT ni.*
+ FROM network_intercepts as ni
+ JOIN runs as r on r.id = ni.run_id
+ WHERE
+ (TRUE = %s OR r.user_id = ANY(%s::int[]))
+ AND ni.run_id = %s
+ ORDER BY ni.time
+ '''
+ params = (user.is_super_user(), user.visible_users(), run_id)
+ data = DB().fetch_all(query, params=params)
+
+ escaped_data = html_escape_multi(data)
+ return ORJSONResponseObjKeep({'success': True, 'data': escaped_data})
+
+
+@router.get('/v1/repositories')
+async def get_repositories(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, sort_by: str = 'name', user: User = Depends(authenticate)):
+ query = '''
+ SELECT
+ r.uri,
+ MAX(r.created_at) as last_run
+ FROM runs as r
+ LEFT JOIN machines as m on r.machine_id = m.id
+ WHERE
+ (TRUE = %s OR r.user_id = ANY(%s::int[]))
+ '''
+
+ params = [user.is_super_user(), user.visible_users()]
+
+ if uri:
+ query = f"{query} AND r.uri LIKE %s \n"
+ params.append(f"%{uri}%")
+
+ if branch:
+ query = f"{query} AND r.branch LIKE %s \n"
+ params.append(f"%{branch}%")
+
+ if filename:
+ query = f"{query} AND r.filename LIKE %s \n"
+ params.append(f"%{filename}%")
+
+ if machine_id and check_int_field_api(machine_id, 'machine_id', 1024):
+ query = f"{query} AND m.id = %s \n"
+ params.append(machine_id)
+
+ if machine:
+ query = f"{query} AND m.description LIKE %s \n"
+ params.append(f"%{machine}%")
+
+ query = f"{query} GROUP BY r.uri\n"
+
+ if sort_by == 'name':
+ query = f"{query} ORDER BY r.uri ASC"
+ else:
+ query = f"{query} ORDER BY last_run DESC"
+
+ data = DB().fetch_all(query, params=params)
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ escaped_data = [html_escape_multi(run) for run in data]
+
+ return ORJSONResponse({'success': True, 'data': escaped_data})
+
+
+# A route to return all of the available entries in our catalog.
+@router.get('/v1/runs')
+async def get_runs(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, limit: int = 5, uri_mode = 'none', user: User = Depends(authenticate)):
+
+ query = '''
+ SELECT r.id, r.name, r.uri, r.branch, r.created_at, r.invalid_run, r.filename, m.description, r.commit_hash, r.end_measurement, r.failed, r.machine_id
+ FROM runs as r
+ LEFT JOIN machines as m on r.machine_id = m.id
+ WHERE
+ (TRUE = %s OR r.user_id = ANY(%s::int[]))
+ '''
+ params = [user.is_super_user(), user.visible_users()]
+
+ if uri:
+ if uri_mode == 'exact':
+ query = f"{query} AND r.uri = %s \n"
+ params.append(uri)
+ else:
+ query = f"{query} AND r.uri LIKE %s \n"
+ params.append(f"%{uri}%")
+
+ if branch:
+ query = f"{query} AND r.branch LIKE %s \n"
+ params.append(f"%{branch}%")
+
+ if filename:
+ query = f"{query} AND r.filename LIKE %s \n"
+ params.append(f"%{filename}%")
+
+ if machine_id and check_int_field_api(machine_id, 'machine_id', 1024):
+ query = f"{query} AND m.id = %s \n"
+ params.append(machine_id)
+
+ if machine:
+ query = f"{query} AND m.description LIKE %s \n"
+ params.append(f"%{machine}%")
+
+ query = f"{query} ORDER BY r.created_at DESC"
+
+ check_int_field_api(limit, 'limit', 50)
+ query = f"{query} LIMIT %s"
+ params.append(limit)
+
+
+ data = DB().fetch_all(query, params=params)
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ escaped_data = [html_escape_multi(run) for run in data]
+
+ return ORJSONResponse({'success': True, 'data': escaped_data})
+
+
+# Just copy and paste if we want to deprecate URLs
+# @router.get('/v1/measurements/uri', deprecated=True) # Here you can see, that URL is nevertheless accessible as variable
+# later if supplied. Also deprecation shall be used once we move to v2 for all v1 routesthrough
+
+@router.get('/v1/compare')
+async def compare_in_repo(ids: str, user: User = Depends(authenticate)):
+ if ids is None or not ids.strip():
+ raise RequestValidationError('run_id is empty')
+ ids = ids.split(',')
+ if not all(is_valid_uuid(id) for id in ids):
+ raise RequestValidationError('One of Run IDs is not a valid UUID or empty')
+
+
+ if artifact := get_artifact(ArtifactType.COMPARE, f"{user._id}_{str(ids)}"):
+ return ORJSONResponse({'success': True, 'data': orjson.loads(artifact)}) # pylint: disable=no-member
+
+ try:
+ case, comparison_db_key = determine_comparison_case(user, ids)
+ except RuntimeError as exc:
+ raise RequestValidationError(str(exc)) from exc
+
+ comparison_details = get_comparison_details(user, ids, comparison_db_key)
+
+ if not (phase_stats := get_phase_stats(user, ids)):
+ return Response(status_code=204) # No-Content
+
+ try:
+ phase_stats_object = get_phase_stats_object(phase_stats, case, comparison_details)
+ phase_stats_object = add_phase_stats_statistics(phase_stats_object)
+ except ValueError as exc:
+ raise RequestValidationError(str(exc)) from exc
+
+ phase_stats_object['common_info'] = {}
+
+ try:
+ run_info = get_run_info(user, ids[0])
+
+ machine_list = get_machine_list()
+ machines = {machine[0]: machine[1] for machine in machine_list}
+
+ machine = machines[run_info['machine_id']]
+ uri = run_info['uri']
+ usage_scenario = run_info['usage_scenario']['name']
+ branch = run_info['branch']
+ commit = run_info['commit_hash']
+ filename = run_info['filename']
+
+ match case:
+ case 'Repeated Run':
+ # same repo, same usage scenarios, same machines, same branches, same commit hashes
+ phase_stats_object['common_info']['Repository'] = uri
+ phase_stats_object['common_info']['Filename'] = filename
+ phase_stats_object['common_info']['Usage Scenario'] = usage_scenario
+ phase_stats_object['common_info']['Machine'] = machine
+ phase_stats_object['common_info']['Branch'] = branch
+ phase_stats_object['common_info']['Commit'] = commit
+ case 'Usage Scenario':
+ # same repo, diff usage scenarios, same machines, same branches, same commit hashes
+ phase_stats_object['common_info']['Repository'] = uri
+ phase_stats_object['common_info']['Machine'] = machine
+ phase_stats_object['common_info']['Branch'] = branch
+ phase_stats_object['common_info']['Commit'] = commit
+ case 'Machine':
+ # same repo, same usage scenarios, diff machines, same branches, same commit hashes
+ phase_stats_object['common_info']['Repository'] = uri
+ phase_stats_object['common_info']['Filename'] = filename
+ phase_stats_object['common_info']['Usage Scenario'] = usage_scenario
+ phase_stats_object['common_info']['Branch'] = branch
+ phase_stats_object['common_info']['Commit'] = commit
+ case 'Commit':
+ # same repo, same usage scenarios, same machines, diff commit hashes
+ phase_stats_object['common_info']['Repository'] = uri
+ phase_stats_object['common_info']['Filename'] = filename
+ phase_stats_object['common_info']['Usage Scenario'] = usage_scenario
+ phase_stats_object['common_info']['Machine'] = machine
+ case 'Repository':
+ # diff repo, diff usage scenarios, same machine, same branches, diff/same commits_hashes
+ phase_stats_object['common_info']['Machine'] = machine
+ phase_stats_object['common_info']['Branch'] = branch
+ case 'Branch':
+ # same repo, same usage scenarios, same machines, diff branch
+ phase_stats_object['common_info']['Repository'] = uri
+ phase_stats_object['common_info']['Filename'] = filename
+ phase_stats_object['common_info']['Usage Scenario'] = usage_scenario
+ phase_stats_object['common_info']['Machine'] = machine
+
+ except RuntimeError as err:
+ raise RequestValidationError(str(err)) from err
+
+ store_artifact(ArtifactType.COMPARE, f"{user._id}_{str(ids)}", orjson.dumps(phase_stats_object)) # pylint: disable=no-member
+
+
+ return ORJSONResponse({'success': True, 'data': phase_stats_object})
+
+
+@router.get('/v1/phase_stats/single/{run_id}')
+async def get_phase_stats_single(run_id: str, user: User = Depends(authenticate)):
+ if run_id is None or not is_valid_uuid(run_id):
+ raise RequestValidationError('Run ID is not a valid UUID or empty')
+
+ if artifact := get_artifact(ArtifactType.STATS, f"{user._id}_{str(run_id)}"):
+ return ORJSONResponse({'success': True, 'data': orjson.loads(artifact)}) # pylint: disable=no-member
+
+ if not (phase_stats := get_phase_stats(user, [run_id])):
+ return Response(status_code=204) # No-Content
+
+ try:
+ phase_stats_object = get_phase_stats_object(phase_stats, None, None, [run_id])
+ phase_stats_object = add_phase_stats_statistics(phase_stats_object)
+ except ValueError as exc:
+ raise RequestValidationError(str(exc)) from exc
+
+ store_artifact(ArtifactType.STATS, f"{user._id}_{str(run_id)}", orjson.dumps(phase_stats_object)) # pylint: disable=no-member
+
+ return ORJSONResponseObjKeep({'success': True, 'data': phase_stats_object})
+
+
+# This route gets the measurements to be displayed in a timeline chart
+@router.get('/v1/measurements/single/{run_id}')
+async def get_measurements_single(run_id: str, user: User = Depends(authenticate)):
+ if run_id is None or not is_valid_uuid(run_id):
+ raise RequestValidationError('Run ID is not a valid UUID or empty')
+
+ query = '''
+ SELECT
+ mm.detail_name, mv.time, mm.metric,
+ mv.value, mm.unit
+ FROM measurement_metrics as mm
+ JOIN measurement_values as mv ON mv.measurement_metric_id = mm.id
+ JOIN runs as r ON mm.run_id = r.id
+ WHERE
+ (TRUE = %s OR r.user_id = ANY(%s::int[]))
+ AND mm.run_id = %s
+ '''
+
+ params = (user.is_super_user(), user.visible_users(), run_id)
+
+ # extremely important to order here, cause the charting library in JS cannot do that automatically!
+ # Furthermore we do time-lag caclulations and need the order of metric first and then time in stats.js:179... . Please do not change
+ query = f"{query} ORDER BY mm.metric ASC, mm.detail_name ASC, mv.time ASC"
+
+ data = DB().fetch_all(query, params=params)
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponseObjKeep({'success': True, 'data': data})
+
+@router.get('/v1/timeline')
+async def get_timeline_stats(uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, start_date: date | None = None, end_date: date | None = None, metrics: str | None = None, phase: str | None = None, sorting: str | None = None, user: User = Depends(authenticate)):
+ if uri is None or uri.strip() == '':
+ raise RequestValidationError('URI is empty')
+
+ if phase is None or phase.strip() == '':
+ raise RequestValidationError('Phase is empty')
+
+ query, params = get_timeline_query(user, uri, filename, machine_id, branch, metrics, phase, start_date=start_date, end_date=end_date, sorting=sorting)
+
+ data = DB().fetch_all(query, params=params)
+
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data})
+
+# Show the timeline badges with regression trend
+## A complex case to allow public visibility of the badge but restricting everything else would be to have
+## User 1 restricted to only this route but a fully populated 'visible_users' array
+@router.get('/v1/badge/timeline')
+async def get_timeline_badge(detail_name: str, uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, metrics: str | None = None, unit: str = 'watt-hours', user: User = Depends(authenticate)):
+ if uri is None or uri.strip() == '':
+ raise RequestValidationError('URI is empty')
+
+ if detail_name is None or detail_name.strip() == '':
+ raise RequestValidationError('Detail Name is mandatory')
+
+ if unit not in ('watt-hours', 'joules'):
+ raise RequestValidationError('Requested unit is not in allow list: watt-hours, joules')
+
+ # we believe that there is no injection possible to the artifact store and any string can be constructured here ...
+ if artifact := get_artifact(ArtifactType.BADGE, f"{user._id}_{uri}_{filename}_{machine_id}_{branch}_{metrics}_{detail_name}_{unit}"):