Wire Solid Queue and add jwt/webmock gems

index 411f0ac..9792d4d 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,9 @@ gem "redcarpet", "~> 3.6" # Syntax highlighting for code files gem "rouge", "~> 4.5" +# RS256 JWT minting for GitHub App authentication (installation access tokens) +gem "jwt", "~> 3.1" + # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword] # gem "bcrypt", "~> 3.1.7" @@ -77,6 +80,9 @@ group :development, :test do # RSpec for request/model specs [https://github.com/rspec/rspec-rails] gem "rspec-rails", "~> 8.0" + + # Stub outbound HTTP (GitHub REST) in specs + gem "webmock", require: false end group :development do

index fae165b..9b69aee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -94,6 +94,9 @@ GEM logger (~> 1.5) concurrent-ruby (1.3.7) connection_pool (3.0.2) + crack (1.0.1) + bigdecimal + rexml crass (1.0.6) date (3.5.1) debug (1.11.1) @@ -111,11 +114,13 @@ GEM et-orbi (1.4.0) tzinfo ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86_64-linux-gnu) fugit (1.12.2) et-orbi (~> 1.4) raabro (~> 1.4) globalid (1.3.0) activesupport (>= 6.1) + hashdiff (1.2.1) i18n (1.15.1) concurrent-ruby (~> 1.0) image_processing (1.14.0) @@ -135,6 +140,8 @@ GEM actionview (>= 7.0.0) activesupport (>= 7.0.0) json (2.19.9) + jwt (3.2.0) + base64 kamal (2.12.0) activesupport (>= 7.0) base64 (~> 0.2) @@ -189,12 +196,15 @@ GEM nio4r (2.7.5) nokogiri (1.19.4-arm64-darwin) racc (~> 1.4) + nokogiri (1.19.4-x86_64-linux-gnu) + racc (~> 1.4) ostruct (0.6.3) parallel (2.1.0) parser (3.3.11.1) ast (~> 2.4.1) racc pg (1.6.3-arm64-darwin) + pg (1.6.3-x86_64-linux) pp (0.6.3) prettyprint prettyprint (0.2.0) @@ -262,6 +272,7 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) + rexml (3.4.4) rouge (4.7.0) rspec-core (3.13.6) rspec-support (~> 3.13.0) @@ -347,8 +358,10 @@ GEM railties (>= 7.0.0) tailwindcss-ruby (~> 3.0) tailwindcss-ruby (3.4.19-arm64-darwin) + tailwindcss-ruby (3.4.19-x86_64-linux) thor (1.5.0) thruster (0.1.21-arm64-darwin) + thruster (0.1.21-x86_64-linux) timeout (0.6.1) tsort (0.2.0) turbo-rails (2.0.23) @@ -365,6 +378,10 @@ GEM actionview (>= 8.0.0) bindex (>= 0.4.0) railties (>= 8.0.0) + webmock (3.26.2) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) websocket-driver (0.8.1) base64 websocket-extensions (>= 0.1.0) @@ -373,6 +390,7 @@ GEM PLATFORMS arm64-darwin + x86_64-linux DEPENDENCIES bootsnap @@ -383,6 +401,7 @@ DEPENDENCIES image_processing (~> 1.2) importmap-rails jbuilder + jwt (~> 3.1) kamal letter_opener pg (~> 1.1) @@ -404,6 +423,7 @@ DEPENDENCIES turbo-rails tzinfo-data web-console + webmock BUNDLED WITH 2.7.2

new file mode 100755 index 0000000..dcf59f3 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV)

index cb0241d..04aade1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -50,7 +50,8 @@ Rails.application.configure do # config.cache_store = :mem_cache_store # Replace the default in-process and non-durable queuing backend for Active Job. - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors.

new file mode 100644 index 0000000..336f5dd --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,20 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + # Each review job can pin a thread on an Onde Cloud request for up to + # ~90s (OndeCloudService::READ_TIMEOUT) — keep concurrency low. + threads: 2 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default

new file mode 100644 index 0000000..b4207f9 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,15 @@ +# examples: +# periodic_cleanup: +# class: CleanSoftDeletedRecordsJob +# queue: background +# args: [ 1000, { batch_size: 500 } ] +# schedule: every hour +# periodic_cleanup_with_command: +# command: "SoftDeletedRecord.due.delete_all" +# priority: 2 +# schedule: at 5am every day + +production: + clear_solid_queue_finished_jobs: + command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + schedule: every hour at minute 12

new file mode 100644 index 0000000..85194b6 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,129 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end