Add detailed architecture diagram for BetelgeuseBytes AI platform
This commit is contained in:
parent
404deb1d52
commit
7906e9490f
167
ARCHITECTURE.md
167
ARCHITECTURE.md
|
|
@ -91,3 +91,170 @@ Kubernetes Cluster
|
||||||
* End-to-end ML lifecycle
|
* End-to-end ML lifecycle
|
||||||
* Automated data pipelines
|
* Automated data pipelines
|
||||||
* Production observability-first apps
|
* Production observability-first apps
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
|
||||||
|
flowchart TB
|
||||||
|
%% =========================
|
||||||
|
%% BetelgeuseBytes AI Platform – Full Architecture (CPU-first, K8s)
|
||||||
|
%% =========================
|
||||||
|
|
||||||
|
%% ---- External / Users ----
|
||||||
|
subgraph EXT["External Users & Clients"]
|
||||||
|
U1["Scholar / Admin User\n(Web Browser)"]
|
||||||
|
U2["API Client\n(curl / SDK / Bots)"]
|
||||||
|
U3["Annotator\n(Labeling UI)"]
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ---- DNS + TLS + Ingress ----
|
||||||
|
subgraph EDGE["Edge: DNS → TLS → Ingress"]
|
||||||
|
DNS["DNS: betelgeusebytes.io\nA/AAAA records → Ingress IP"]
|
||||||
|
CM["cert-manager\nLet's Encrypt TLS"]
|
||||||
|
INGRESS["NGINX Ingress Controller\nHTTP(S) + SNI routing"]
|
||||||
|
TCPMAP["Ingress TCP Services\n(Postgres, Neo4j Bolt)"]
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ---- Kubernetes Cluster ----
|
||||||
|
subgraph K8S["Kubernetes Cluster (Hetzner Dedicated • CPU-only • local SSD PVs)"]
|
||||||
|
direction TB
|
||||||
|
|
||||||
|
subgraph NET["Networking"]
|
||||||
|
CILIUM["Cilium CNI\n(eBPF dataplane / policies)"]
|
||||||
|
WG["WireGuard\n(node mesh / private networking)"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph DEVOPS["DevOps / GitOps"]
|
||||||
|
GITEA["Gitea\nGit repos"]
|
||||||
|
ARGOCD["Argo CD\nGitOps deployments"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph OBS["Observability (LGTM)"]
|
||||||
|
ALLOY["Grafana Alloy\n(collect logs+traces)"]
|
||||||
|
PROM["Prometheus\n(metrics)"]
|
||||||
|
LOKI["Loki\n(logs)"]
|
||||||
|
TEMPO["Tempo\n(traces)"]
|
||||||
|
GRAF["Grafana\n(dashboards)"]
|
||||||
|
KSM["kube-state-metrics"]
|
||||||
|
NODEX["node-exporter"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph DATA["Core Data Layer"]
|
||||||
|
PG["PostgreSQL\n(app DB / MLflow / Label Studio)\nNamespace: db"]
|
||||||
|
REDIS["Redis\n(cache)\nNamespace: db"]
|
||||||
|
ES["Elasticsearch\n(search/log store)\nNamespace: elastic"]
|
||||||
|
KIB["Kibana\nUI\nNamespace: elastic"]
|
||||||
|
KAFKA["Kafka\n(event bus)\nNamespace: broker"]
|
||||||
|
KAFKAUI["Kafka UI\nUI\nNamespace: broker"]
|
||||||
|
MINIO["MinIO (S3)\n(datasets & artifacts)\nNamespace: storage"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph KG["Knowledge & Retrieval"]
|
||||||
|
NEO4J["Neo4j\n(knowledge graph)\nNamespace: graph"]
|
||||||
|
QDRANT["Qdrant\n(vector DB + UI)\nNamespace: vec"]
|
||||||
|
TEI["Text Embeddings Inference\n(embeddings API)\nNamespace: ai"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph AI["AI / ML Services"]
|
||||||
|
LLM["LLM Server (CPU)\nOllama / llama.cpp\nNamespace: ai"]
|
||||||
|
JUP["Jupyter\n(research notebooks)\nNamespace: ml"]
|
||||||
|
LABEL["Label Studio\n(annotation UI)\nNamespace: ai"]
|
||||||
|
MLFLOW["MLflow\n(tracking + registry)\nNamespace: mlops/ml"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph PIPE["Automation / Pipelines"]
|
||||||
|
ARGO_WF["Argo Workflows\n(pipelines)\nNamespace: ml/argo"]
|
||||||
|
N8N["n8n\n(automation)\nNamespace: automation"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph AUTH["Authentication"]
|
||||||
|
KEYCLOAK["Keycloak\n(OIDC/SSO)\nNamespace: auth"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph APPS["Custom Applications (to build)"]
|
||||||
|
ORCH["Hadith Orchestrator API\n(FastAPI)\nNamespace: hadith"]
|
||||||
|
ADMIN["Hadith Admin UI\n(Next.js)\nNamespace: hadith"]
|
||||||
|
NER["NER Service\n(custom)\nNamespace: hadith"]
|
||||||
|
RE["Relation Extraction Service\n(custom)\nNamespace: hadith"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ---- Edge wiring ----
|
||||||
|
U1 --> DNS
|
||||||
|
U2 --> DNS
|
||||||
|
U3 --> DNS
|
||||||
|
DNS --> INGRESS
|
||||||
|
CM --> INGRESS
|
||||||
|
|
||||||
|
%% ---- Public HTTP(S) routes ----
|
||||||
|
INGRESS -->|hadith-admin.betelgeusebytes.io| ADMIN
|
||||||
|
INGRESS -->|hadith-api.betelgeusebytes.io| ORCH
|
||||||
|
INGRESS -->|llm.betelgeusebytes.io| LLM
|
||||||
|
INGRESS -->|embeddings.betelgeusebytes.io| TEI
|
||||||
|
INGRESS -->|vector.betelgeusebytes.io| QDRANT
|
||||||
|
INGRESS -->|neo4j.betelgeusebytes.io| NEO4J
|
||||||
|
INGRESS -->|label.betelgeusebytes.io| LABEL
|
||||||
|
INGRESS -->|mlflow.betelgeusebytes.io| MLFLOW
|
||||||
|
INGRESS -->|minio.betelgeusebytes.io| MINIO
|
||||||
|
INGRESS -->|argo.betelgeusebytes.io| ARGO_WF
|
||||||
|
INGRESS -->|auth.betelgeusebytes.io| KEYCLOAK
|
||||||
|
INGRESS -->|grafana.betelgeusebytes.io| GRAF
|
||||||
|
INGRESS -->|kibana.betelgeusebytes.io| KIB
|
||||||
|
INGRESS -->|broker.betelgeusebytes.io| KAFKAUI
|
||||||
|
|
||||||
|
%% ---- TCP routes (optional/external) ----
|
||||||
|
TCPMAP -.-> PG
|
||||||
|
TCPMAP -.-> NEO4J
|
||||||
|
|
||||||
|
%% ---- GitOps flow ----
|
||||||
|
GITEA -->|manifests + app code| ARGOCD
|
||||||
|
ARGOCD -->|sync/apply| K8S
|
||||||
|
|
||||||
|
%% ---- Auth flows ----
|
||||||
|
ADMIN -->|OIDC login| KEYCLOAK
|
||||||
|
ORCH -->|validate JWT / introspect| KEYCLOAK
|
||||||
|
LABEL -->|optional OIDC| KEYCLOAK
|
||||||
|
MLFLOW -->|OIDC| KEYCLOAK
|
||||||
|
|
||||||
|
%% ---- Orchestrator runtime data flows ----
|
||||||
|
ORCH -->|reasoning / JSON extraction| LLM
|
||||||
|
ORCH -->|embed queries/docs| TEI
|
||||||
|
ORCH -->|vector search| QDRANT
|
||||||
|
ORCH -->|graph read/write| NEO4J
|
||||||
|
ORCH -->|metadata/users/jobs| PG
|
||||||
|
ORCH -->|cache| REDIS
|
||||||
|
ORCH -->|full-text search| ES
|
||||||
|
|
||||||
|
%% ---- NER/RE services (future) ----
|
||||||
|
ORCH --> NER
|
||||||
|
ORCH --> RE
|
||||||
|
NER -->|entities| NEO4J
|
||||||
|
RE -->|relations| NEO4J
|
||||||
|
|
||||||
|
%% ---- Data curation loop ----
|
||||||
|
LABEL -->|labeled datasets| MINIO
|
||||||
|
ARGO_WF -->|training data| MINIO
|
||||||
|
ARGO_WF -->|log metrics| MLFLOW
|
||||||
|
ARGO_WF -->|publish artifacts| MINIO
|
||||||
|
MLFLOW -->|model versions| MINIO
|
||||||
|
ARGO_WF -->|deploy/update services| ARGOCD
|
||||||
|
|
||||||
|
%% ---- Event-driven (optional) ----
|
||||||
|
ORCH -->|events| KAFKA
|
||||||
|
ARGO_WF -->|consume triggers| KAFKA
|
||||||
|
N8N -->|integrations/alerts| KAFKA
|
||||||
|
|
||||||
|
%% ---- Observability wiring ----
|
||||||
|
ALLOY --> LOKI
|
||||||
|
ALLOY --> TEMPO
|
||||||
|
PROM --> GRAF
|
||||||
|
LOKI --> GRAF
|
||||||
|
TEMPO --> GRAF
|
||||||
|
KSM --> PROM
|
||||||
|
NODEX --> PROM
|
||||||
|
|
||||||
|
%% ---- Internal networking ----
|
||||||
|
CILIUM --- INGRESS
|
||||||
|
WG --- CILIUM
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue