torbo/flyer-crawler.projectium.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

db optimizzzation, error handling etc
Some checks failed
Deploy to Web Server flyer-crawler.projectium.com / deploy (push) Failing after 3m48s
Details

Browse Source
This commit is contained in:
Torben Sorensen
2025-11-30 12:37:47 -08:00
parent c45fdc6c30
commit 42e4a308e4
4 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.gitea/workflows/deploy.yml
View File

@@ -209,8 +209,8 @@ jobs:
# Query the production database to get the hash of the deployed schema.
# The `psql` command requires PGPASSWORD to be set.
# `\t` sets tuples-only mode and `\A` unaligns output to get just the raw value.
# The `|| echo "none"` ensures the command doesn't fail if the table or row doesn't exist yet.
DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
# The `|| echo "none"` ensures the command doesn't fail if the table or row doesn't exist yet.
DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
echo "Deployed DB Schema Hash: $DEPLOYED_HASH"
# Check if the hash is "none" (command failed) OR if it's an empty string (table exists but is empty).
@@ -291,13 +291,13 @@ jobs:
# After a successful deployment, update the schema hash in the database.
# This ensures the next deployment will compare against this new state.
echo "Updating schema hash in production database..."
CURRENT_HASH=$(cat sql/master_schema_rollup.sql | dos2unix | sha256sum | awk '{ print $1 }')
PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
CURRENT_HASH=$(cat sql/master_schema_rollup.sql | dos2unix | sha256sum | awk '{ print $1 }')
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
"INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"
# Verify the hash was updated
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
echo "✅ Schema hash successfully updated in the database to: $UPDATED_HASH"
else

14
.gitea/workflows/manual-db-reset.yml
View File

@@ -66,7 +66,7 @@ jobs:
echo "Attempting to back up data for user: $USER_EMAIL"
# Get the user_id for the specified email.
USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)
USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)
if [ -z "$USER_ID" ]; then
echo "WARNING: User with email '$USER_EMAIL' not found. Skipping backup."
@@ -99,20 +99,20 @@ jobs:
- name: Step 2 - Drop All Tables from Production DB
run: |
echo "Executing drop_tables.sql against the PRODUCTION database..."
PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/drop_tables.sql
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/drop_tables.sql
echo "✅ All tables dropped successfully."
- name: Step 3 - Rebuild Schema from Master Rollup
run: |
echo "Executing master_schema_rollup.sql against the PRODUCTION database..."
PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/master_schema_rollup.sql
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/master_schema_rollup.sql
echo "✅ Schema rebuilt successfully."
- name: Step 4 - (Optional) Restore Specific User Data
if: ${{ gitea.event.inputs.user_email != '' && env.NO_USER_BACKUP != 'true' }}
run: |
echo "Restoring user data from filtered_backup.sql..."
PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f filtered_backup.sql
echo "Restoring user data from filtered_backup.sql..."
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f filtered_backup.sql
echo "✅ User data restored successfully."
- name: Step 5 - Update Schema Info Table
@@ -123,12 +123,12 @@ jobs:
echo "New Schema Hash: $CURRENT_HASH"
# Insert the new hash into the freshly created schema_info table.
PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
"INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"
# Verify the hash was updated
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
echo "✅ Schema hash successfully set in the database to: $UPDATED_HASH"
else