phoenix/soaricarus_api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Obj str do it (#255)
soaricarus_api CI / Clippy (push) Successful in 3m59s
Details
soaricarus_api CI / Rustfmt (push) Successful in 4m32s
Details
soaricarus_api CI / Check (push) Successful in 4m44s
Details
soaricarus_api CI / Test Suite (push) Failing after 5m24s
Details
soaricarus_api CI / build (push) Successful in 6m4s
Details

Browse Source 
Reviewed-on: #255
This commit was merged in pull request #255.
This commit is contained in:
phoenix
2026-08-08 14:29:14 -04:00
parent 59a7e31641
commit 19d6ec2190
10 changed files with 1401 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.docker.sample
+21
View File
@@ -11,3 +11,24 @@ POSTGRES_MAIN_DB=soaricarus_db
POSTGRES_MAIN_HOST=main_db
POSTGRES_MAIN_PORT=5432
DATABASE_URL=postgres://${POSTGRES_MAIN_USER}:${POSTGRES_MAIN_PASSWORD}@${POSTGRES_MAIN_HOST}:${POSTGRES_MAIN_PORT}/${POSTGRES_MAIN_DB}
S3_BUCKET_NAME=soaricarus-bucket
S3_ENDPOINT_URL=http://maze:3900
GARAGE_S3_REGION=maze
GARAGE_RPC_SECRET=c2bde5c3f9c04116a7563e25ea5ba6fab25319daa2fe8a952ad57d0b8159b590
GARAGE_METRICS_TOKEN=sD2g4jiYV66KdZWtIuj2amezuryzv81mZNktpMNcmU4=
GARAGE_DEFAULT_ACCESS_KEY=GK8eb91557b9a18dcbdc834c16a25ac6fc
GARAGE_DEFAULT_SECRET_KEY=WhFAtnAtWgYaa0pMfl9jwygalZSwYaVliY3Ndm7nYHo=
GARAGE_DEFAULT_BUCKET=${S3_BUCKET_NAME}
GARAGE_ADMIN_TOKEN=5yMAjSINa6U04wr4pDbcTYfsXKl3S9ZjKdiPXWYUbNw=
GARAGE_METADATA_DIR=/var/lib/garage/meta
GARAGE_DATA_DIR=/var/lib/garage/data
GARAGE_DB_ENGINE=sqlite
GARAGE_REPLICATION_FACTOR=1
GARAGE_RPC_BIND_ADDR=[::]:3901
GARAGE_S3_API_BIND_ADDR=[::]:3900
GARAGE_S3_WEB_BIND_ADDR=[::]:3902
GARAGE_ROOT_DOMAIN=.localhost
AWS_DEFAULT_REGION=${GARAGE_S3_REGION}
AWS_SECRET_ACCESS_KEY=${GARAGE_DEFAULT_ACCESS_KEY}
AWS_ACCESS_KEY_ID=${GARAGE_DEFAULT_SECRET_KEY}
.env.local.sample
+21
View File
@@ -11,3 +11,24 @@ POSTGRES_MAIN_DB=soaricarus_db
POSTGRES_MAIN_HOST=main_db
POSTGRES_MAIN_PORT=5432
DATABASE_URL=postgres://${POSTGRES_MAIN_USER}:${POSTGRES_MAIN_PASSWORD}@${POSTGRES_MAIN_HOST}:${POSTGRES_MAIN_PORT}/${POSTGRES_MAIN_DB}
S3_BUCKET_NAME=soaricarus-bucket
S3_ENDPOINT_URL=http://maze:3900
GARAGE_S3_REGION=maze
GARAGE_RPC_SECRET=c2bde5c3f9c04116a7563e25ea5ba6fab25319daa2fe8a952ad57d0b8159b590
GARAGE_METRICS_TOKEN=sD2g4jiYV66KdZWtIuj2amezuryzv81mZNktpMNcmU4=
GARAGE_DEFAULT_ACCESS_KEY=GK8eb91557b9a18dcbdc834c16a25ac6fc
GARAGE_DEFAULT_SECRET_KEY=WhFAtnAtWgYaa0pMfl9jwygalZSwYaVliY3Ndm7nYHo=
GARAGE_DEFAULT_BUCKET=${S3_BUCKET_NAME}
GARAGE_ADMIN_TOKEN=5yMAjSINa6U04wr4pDbcTYfsXKl3S9ZjKdiPXWYUbNw=
GARAGE_METADATA_DIR=/var/lib/garage/meta
GARAGE_DATA_DIR=/var/lib/garage/data
GARAGE_DB_ENGINE=sqlite
GARAGE_REPLICATION_FACTOR=1
GARAGE_RPC_BIND_ADDR=[::]:3901
GARAGE_S3_API_BIND_ADDR=[::]:3900
GARAGE_S3_WEB_BIND_ADDR=[::]:3902
GARAGE_ROOT_DOMAIN=.localhost
AWS_DEFAULT_REGION=${GARAGE_S3_REGION}
AWS_SECRET_ACCESS_KEY=${GARAGE_DEFAULT_ACCESS_KEY}
AWS_ACCESS_KEY_ID=${GARAGE_DEFAULT_SECRET_KEY}
.github/workflows/pr.yml
+116
View File
@@ -0,0 +1,116 @@
name: pr CI
on:
pull_request:
branches: [ "main" ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
name: Test Suite
runs-on: ubuntu-24.04
timeout-minutes: 20
services:
postgres:
image: postgres:18.4-alpine
env:
POSTGRES_USER: ${{ secrets.DB_TEST_USER || 'testuser' }}
POSTGRES_PASSWORD: ${{ secrets.DB_TEST_PASSWORD || 'testpassword' }}
POSTGRES_DB: ${{ secrets.DB_TEST_NAME || 'testdb' }}
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v6
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: 1.97
- uses: Swatinem/rust-cache@v2
- name: Verify Docker Environment
run: |
echo "Runner User Info:"
id
echo "Checking Docker Version:"
docker --version
echo "Checking Docker Daemon Status (info):"
docker info
echo "Checking Docker Daemon Status (ps):"
docker ps -a
echo "Docker environment check complete."
- name: Setup S3
env:
AWS_ACCESS_KEY_ID: "${{ secrets.GARAGE_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.GARAGE_SECRET }}"
AWS_DEFAULT_REGION: us-east-1
ENDPOINT_URL: "${{ secrets.S3_CI_ENDPOINT_URL}}"
run: |
sudo apt update
echo "Installing curl and unzip"
sudo apt install curl unzip -y
echo "Installing awscli"
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
BUCKET="soaricarus-ci-bucket-$(echo $RANDOM | md5sum | head -c 8)"
echo "S3_BUCKET=$BUCKET" >> $GITHUB_ENV
echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV
echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV
echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV
echo "Creating bucket: $BUCKET"
aws --endpoint-url=$ENDPOINT_URL s3 mb s3://$BUCKET --region $AWS_DEFAULT_REGION
- name: Run tests
env:
DATABASE_URL: postgresql://${{ secrets.DB_TEST_USER || 'testuser' }}:${{ secrets.DB_TEST_PASSWORD || 'testpassword' }}@postgres:5432/${{ secrets.DB_TEST_NAME || 'testdb' }}
RUST_LOG: info # Optional: configure test log level
SECRET_MAIN_KEY: ${{ secrets.TOKEN_SECRET_KEY }}
SSH_AUTH_SOCK: ${{ env.SSH_AUTH_SOCK }}
ROOT_DIRECTORY: "/tmp"
AWS_ACCESS_KEY_ID: "${{ secrets.GARAGE_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.GARAGE_SECRET }}"
AWS_DEFAULT_REGION: "${{ env.AWS_DEFAULT_REGION }}"
GARAGE_DEFAULT_ACCESS_KEY: "${{ secrets.GARAGE_KEY_ID }}"
GARAGE_DEFAULT_SECRET_KEY: "${{ secrets.GARAGE_SECRET }}"
GARAGE_S3_REGION: "${{ env.AWS_DEFAULT_REGION }}"
S3_BUCKET_NAME: ${{ env.S3_BUCKET }}
S3_ENDPOINT_URL: "${{ secrets.S3_CI_ENDPOINT_URL }}"
AWS_S3_FORCE_PATH_STYLE: "true"
run: |
mkdir -p ~/.ssh
echo "${{ secrets.MYREPO_TOKEN }}" > ~/.ssh/soaricarus_models_deploy_key
chmod 600 ~/.ssh/soaricarus_models_deploy_key
ssh-keyscan ${{ secrets.MYHOST }} >> ~/.ssh/known_hosts
eval $(ssh-agent -s)
ssh-add -v ~/.ssh/soaricarus_models_deploy_key
cargo test
- name: Cleanup
if: always()
run: |
echo "Cleaning up bucket $BUCK"
echo "Region: $AWS_DEFAULT_REGION"
aws --endpoint-url=$ENDPOINT_URL \
s3 rb s3://$BUCK --force \
--region $AWS_DEFAULT_REGION
env:
ENDPOINT_URL: ${{ secrets.S3_CI_ENDPOINT_URL }}
AWS_ACCESS_KEY_ID: "${{ env.AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ env.AWS_SECRET_ACCESS_KEY }}"
AWS_DEFAULT_REGION: "${{ env.AWS_DEFAULT_REGION }}"
BUCK: ${{ env.S3_BUCKET }}
continue-on-error: true
.github/workflows/workflow.yml
+54 -3
View File
@@ -3,8 +3,6 @@ name: soaricarus_api CI
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -35,6 +33,7 @@ jobs:
test:
name: Test Suite
runs-on: ubuntu-24.04
timeout-minutes: 20
services:
postgres:
image: postgres:18.4-alpine
@@ -65,14 +64,49 @@ jobs:
echo "Checking Docker Daemon Status (ps):"
docker ps -a
echo "Docker environment check complete."
- name: Setup S3
env:
AWS_ACCESS_KEY_ID: "${{ secrets.GARAGE_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.GARAGE_SECRET }}"
AWS_DEFAULT_REGION: us-east-1
ENDPOINT_URL: "${{ secrets.S3_CI_ENDPOINT_URL}}"
run: |
sudo apt update
echo "Installing curl and unzip"
sudo apt install curl unzip -y
echo "Installing awscli"
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
BUCKET="soaricarus-ci-bucket-$(echo $RANDOM | md5sum | head -c 8)"
echo "S3_BUCKET=$BUCKET" >> $GITHUB_ENV
echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV
echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV
echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV
echo "Creating bucket: $BUCKET"
aws --endpoint-url=$ENDPOINT_URL s3 mb s3://$BUCKET --region $AWS_DEFAULT_REGION
- name: Run tests
env:
# Define DATABASE_URL for tests to use
DATABASE_URL: postgresql://${{ secrets.DB_TEST_USER || 'testuser' }}:${{ secrets.DB_TEST_PASSWORD || 'testpassword' }}@postgres:5432/${{ secrets.DB_TEST_NAME || 'testdb' }}
RUST_LOG: info # Optional: configure test log level
SECRET_MAIN_KEY: ${{ secrets.TOKEN_SECRET_KEY }}
SSH_AUTH_SOCK: ${{ env.SSH_AUTH_SOCK }}
ROOT_DIRECTORY: "/tmp"
AWS_ACCESS_KEY_ID: ${{ secrets.GARAGE_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.GARAGE_SECRET }}
AWS_DEFAULT_REGION: "${{ env.AWS_DEFAULT_REGION }}"
GARAGE_DEFAULT_ACCESS_KEY: ${{ secrets.GARAGE_KEY_ID }}
GARAGE_DEFAULT_SECRET_KEY: ${{ secrets.GARAGE_SECRET }}
GARAGE_S3_REGION: "${{ env.AWS_DEFAULT_REGION }}"
S3_BUCKET_NAME: "soaricarus-ci-bucket"
S3_ENDPOINT_URL: "http://localhost:3900"
AWS_S3_FORCE_PATH_STYLE: "true"
run: |
mkdir -p ~/.ssh
echo "${{ secrets.MYREPO_TOKEN }}" > ~/.ssh/soaricarus_models_deploy_key
@@ -84,6 +118,23 @@ jobs:
cargo test
- name: Cleanup
if: always()
run: |
echo "Cleaning up bucket $BUCK"
echo "Region: $AWS_DEFAULT_REGION"
aws --endpoint-url=$ENDPOINT_URL \
s3 rb s3://$BUCK --force \
--region $AWS_DEFAULT_REGION
env:
ENDPOINT_URL: ${{ secrets.S3_CI_ENDPOINT_URL }}
AWS_ACCESS_KEY_ID: "${{ env.AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ env.AWS_SECRET_ACCESS_KEY }}"
AWS_DEFAULT_REGION: "${{ env.AWS_DEFAULT_REGION }}"
BUCK: ${{ env.S3_BUCKET }}
continue-on-error: true
fmt:
name: Rustfmt
runs-on: ubuntu-24.04
Cargo.lock
Generated
+1014 -35
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "soaricarus_api"
version = "0.5.2"
version = "0.5.3"
edition = "2024"
rust-version = "1.95"
license = "MIT"
@@ -29,6 +29,7 @@ utoipa-swagger-ui = { version = "9.0.2", features = ["axum"] }
simeta = { git = "ssh://git@git.kundeng.us/phoenix/simeta.git", tag = "v0.6.1-main-b690995806-680" }
simodels = { git = "ssh://git@git.kundeng.us/phoenix/simodels.git", tag = "v0.11.3-main-fe9d101bd0-111" }
sienvy = { git = "ssh://git@git.kundeng.us/phoenix/sienvy.git", tag = "v0.8.0-main-d06c8fdf49-006" }
labyrinth = { git = "ssh://git@git.kundeng.us/phoenix/labyrinth.git", tag = "v0.0.2-1-84f98521c8-928" }
[dev-dependencies]
common-multipart-rfc7578 = { version = "0.7.0" }
README.md
+15
View File
@@ -19,6 +19,21 @@ Copy the `.env.docker.sample` file to `.env`. Ensure that the `ROOT_DIRECTORY` v
and exists on the docker image's filesystem. The credentials for the database doesn't need to be
changed for development, but if deploying it, it should be modified.
### S3 Setup
Using the copied `.env.docker.sample`, ensure that the following has been properly populated.
```
S3_ENDPOINT_URL - URL of the S3 bucket
S3_BUCKET_NAME - Name of the S3 bucket
GARAGE_S3_REGION - Region of the S3 bucket
GARAGE_RPC_SECRET - A 32-bit hex string
GARAGE_DEFAULT_ACCESS_KEY - Access key starting with GK followed with a 16-bit hex string
GARAGE_DEFAULT_SECRET_KEY - Secret key with a 32-bit hex string
```
The other S3-related environment variables can be left as is for the sake of getting
started.
Build containers
```
docker compose build --ssh default
docker-compose.yaml
+77 -1
View File
@@ -6,14 +6,20 @@ services:
build:
context: .
ssh: ["default"]
container_name: soaricarus
container_name: api
ports:
- "8000:8000"
env_file:
- .env
volumes:
- shared_credentials:/shared:ro # Mount as read-only
depends_on:
main_db:
condition: service_healthy
maze:
condition: service_started
auth_api:
condition: service_started
networks:
- soaricarus-network
restart: unless-stopped
@@ -54,9 +60,75 @@ services:
- main_db
- auth_api
- auth_db
- maze
networks:
- soaricarus-network
# --- storage service ---
maze:
image: dxflrs/garage:v2.3.0 # Use official Garage image
container_name: maze
restart: unless-stopped
environment:
# Garage configuration via environment variables
GARAGE_SERVER_BIND_ADDR: "0.0.0.0:3900"
GARAGE_SERVER_BIND_PUBLIC_ADDR: "0.0.0.0:3901"
GARAGE_S3_API_BIND_ADDR: "0.0.0.0:3900"
GARAGE_S3_API_PUBLIC_BIND_ADDR: "0.0.0.0:3901"
GARAGE_S3_API_PUBLIC_ADDRS: "http://localhost:3901"
GARAGE_S3_API_ROOT_DOMAIN: "${GARAGE_ROOT_DOMAIN}"
GARAGE_ADMIN_BIND_ADDR: "0.0.0.0:3902"
GARAGE_ADMIN_PUBLIC_BIND_ADDR: "0.0.0.0:3902"
# Cluster configuration (single node for simplicity)
GARAGE_CLUSTER_REPLICATION_MODE: "none" # Single node
GARAGE_CLUSTER_PEER_REPLICATION_STRATEGY: "ring"
# Data storage
GARAGE_DATA_DIR: "/data"
GARAGE_METADATA_DIR: "/data/meta"
# RPC configuration
GARAGE_RPC_PUBLIC_ADDR: "http://maze:3901"
GARAGE_RPC_BIND_ADDR: "0.0.0.0:3901"
GARAGE_RPC_SECRET: "${GARAGE_RPC_SECRET}" # 32+ chars
# Web Admin
GARAGE_ADMIN_TOKEN: "${GARAGE_ADMIN_TOKEN}"
# Logging
GARAGE_LOG_LEVEL: "info"
GARAGE_S3_BUCKET: "${S3_BUCKET_NAME}"
GARAGE_S3_REGION: "${GARAGE_S3_REGION}"
GARAGE_DEFAULT_ACCESS_KEY: "${GARAGE_DEFAULT_ACCESS_KEY}"
GARAGE_DEFAULT_SECRET_KEY: "${GARAGE_DEFAULT_SECRET_KEY}"
GARAGE_DEFAULT_BUCKET: "${GARAGE_DEFAULT_BUCKET}"
AWS_ACCESS_KEY_ID: "${GARAGE_DEFAULT_ACCESS_KEY}"
AWS_SECRET_ACCESS_KEY: "${GARAGE_DEFAULT_SECRET_KEY}"
AWS_DEFAULT_REGION: "${GARAGE_DEFAULT_BUCKET}"
AWS_S3_FORCE_PATH_STYLE: "true"
volumes:
- ./garage.toml:/etc/garage.toml:ro # Mount config file
- shared_credentials:/data # Persist Garage data
ports:
- "3900:3900" # S3 API
- "3901:3901" # S3 API public
- "3902:3902" # Admin API
command: ["/garage", "server", "--single-node", "--default-bucket"]
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3900/"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
networks:
- soaricarus-network
extra_hosts:
- "garage.local:127.0.0.1"
# PostgreSQL Database Service
main_db:
@@ -113,6 +185,10 @@ volumes:
driver: local # Use the default local driver
postgres_data_auth:
driver: local # Use the default local driver
object_data:
driver: local # Use the default local driver
shared_credentials: # New shared volume
driver: local
networks:
soaricarus-network:
garage.toml
+24
View File
@@ -0,0 +1,24 @@
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "${GARAGE_RPC_SECRET}"
rpc_bind_addr = "[::]:3901"
[s3_api]
s3_region = "maze"
api_bind_addr = "[::]:3900"
root_domain = ".s3.maze.localhost"
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.maze.localhost"
index = "index.html"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "${GARAGE_ADMIN_TOKEN}"
metrics_token = "${GARAGE_METRICS_TOKEN}"
src/callers/queue/song.rs
+57
View File
@@ -156,6 +156,7 @@ pub mod endpoint {
);
let raw_data: Vec<u8> = data.to_vec();
let copied_raw_data = data.to_vec();
match super::is_song_valid(&raw_data).await {
Ok(valid) => {
if valid {
@@ -169,6 +170,62 @@ pub mod endpoint {
{
Ok(queued_song) => {
results.push(queued_song);
println!("Uploading to bucket");
let s3_endpoint_url =
sienvy::environment::get_env("S3_ENDPOINT_URL");
let bucket_name =
sienvy::environment::get_env("S3_BUCKET_NAME");
let region =
sienvy::environment::get_env("GARAGE_S3_REGION");
let access_key_id = sienvy::environment::get_env(
"GARAGE_DEFAULT_ACCESS_KEY",
);
let secret_key = sienvy::environment::get_env(
"GARAGE_DEFAULT_SECRET_KEY",
);
let lab_config = labyrinth::config::Config {
url: s3_endpoint_url.value,
bucket: bucket_name.value,
region: region.value,
access_key_id: access_key_id.value,
secret_key: secret_key.value,
};
println!("Labyrinth config: {lab_config:?}");
let lr = labyrinth::Labyrinth { config: lab_config };
let data = labyrinth::Data {
raw_data: copied_raw_data,
..Default::default()
};
let filename = simodels::song::generate_filename(
simodels::types::MusicType::FlacExtension,
true,
)
.unwrap();
println!("Filename: {filename:?}");
let file_path = format!("queued/song/{filename}");
println!("Path: {file_path:?}");
match lr.upload(&file_path, &data).await {
Ok(res) => {
println!("Result: {res:?}");
}
Err(err) => match err {
labyrinth::Error::Info(err_str) => {
eprintln!("Error: {:?}", err_str);
response.message = err_str.to_string();
return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(response));
}
labyrinth::Error::SError(err_s) => {
eprintln!("Error: {:?}", err_s);
response.message = err_s.to_string();
return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(response));
}
},
}
}
Err(err) => {
response.message = err.to_string();