From 6bdf6f03b4f0dec3e5545ae88584ef82da4c1a84 Mon Sep 17 00:00:00 2001 From: Benjamin Neff Date: Mon, 31 Oct 2022 00:13:48 +0100 Subject: Cleanup duplicate pods in database The unique index doesn't work when the port is `NULL`. So use `-1` instead for when using the default ports (80/443), as if we would use the real ports, we could still have both 80 and 443 in the database at the same time. --- app/assets/javascripts/app/views/pod_entry_view.js | 2 +- app/models/pod.rb | 24 ++++++++++++--- .../20221030193943_cleanup_duplicate_pods.rb | 36 ++++++++++++++++++++++ spec/factories.rb | 1 + spec/javascripts/app/views/pod_entry_view_spec.js | 3 +- spec/models/pod_spec.rb | 12 +++++++- 6 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 db/migrate/20221030193943_cleanup_duplicate_pods.rb diff --git a/app/assets/javascripts/app/views/pod_entry_view.js b/app/assets/javascripts/app/views/pod_entry_view.js index 112b6708f..5e45267d1 100644 --- a/app/assets/javascripts/app/views/pod_entry_view.js +++ b/app/assets/javascripts/app/views/pod_entry_view.js @@ -32,7 +32,7 @@ app.views.PodEntry = app.views.Base.extend({ has_errors: (this.model.get("status")!=="no_errors"), status_text: Diaspora.I18n.t("admin.pods.states."+this.model.get("status")), pod_url: (this.model.get("ssl") ? "https" : "http") + "://" + this.model.get("host") + - (this.model.get("port") ? ":" + this.model.get("port") : ""), + (this.model.get("port") >= 0 ? ":" + this.model.get("port") : ""), response_time_fmt: this._fmtResponseTime() /* jshint camelcase: true */ }); diff --git a/app/models/pod.rb b/app/models/pod.rb index 1edc4e899..ff61af5b3 100644 --- a/app/models/pod.rb +++ b/app/models/pod.rb @@ -22,7 +22,7 @@ class Pod < ApplicationRecord ConnectionTester::SSLFailure => :ssl_failed, ConnectionTester::HTTPFailure => :http_failed, ConnectionTester::NodeInfoFailure => :version_failed - } + }.freeze # this are only the most common errors, the rest will be +unknown_error+ CURL_ERROR_MAP = { @@ -34,7 +34,13 @@ class Pod < ApplicationRecord redirected_to_other_hostname: :http_failed }.freeze - DEFAULT_PORTS = [URI::HTTP::DEFAULT_PORT, URI::HTTPS::DEFAULT_PORT] + # use -1 as port for default ports + # we can't use the real default port (80/443) because we need to handle them + # like both are the same and not both can exist at the same time. + # we also can't use nil, because databases don't handle NULL in unique indexes + # (except postgres >= 15 with "NULLS NOT DISTINCT"). + DEFAULT_PORT = -1 + DEFAULT_PORTS = [URI::HTTP::DEFAULT_PORT, URI::HTTPS::DEFAULT_PORT].freeze has_many :people @@ -51,7 +57,7 @@ class Pod < ApplicationRecord class << self def find_or_create_by(opts) # Rename this method to not override an AR method uri = URI.parse(opts.fetch(:url)) - port = DEFAULT_PORTS.include?(uri.port) ? nil : uri.port + port = DEFAULT_PORTS.include?(uri.port) ? DEFAULT_PORT : uri.port find_or_initialize_by(host: uri.host, port: port).tap do |pod| pod.ssl ||= (uri.scheme == "https") pod.save @@ -147,13 +153,21 @@ class Pod < ApplicationRecord # @return [URI] def uri - @uri ||= (ssl ? URI::HTTPS : URI::HTTP).build(host: host, port: port) + @uri ||= (ssl ? URI::HTTPS : URI::HTTP).build(host: host, port: real_port) @uri.dup end + def real_port + if port == DEFAULT_PORT + ssl ? URI::HTTPS::DEFAULT_PORT : URI::HTTP::DEFAULT_PORT + else + port + end + end + def not_own_pod pod_uri = AppConfig.pod_uri - pod_port = DEFAULT_PORTS.include?(pod_uri.port) ? nil : pod_uri.port + pod_port = DEFAULT_PORTS.include?(pod_uri.port) ? DEFAULT_PORT : pod_uri.port errors.add(:base, "own pod not allowed") if pod_uri.host == host && pod_port == port end end diff --git a/db/migrate/20221030193943_cleanup_duplicate_pods.rb b/db/migrate/20221030193943_cleanup_duplicate_pods.rb new file mode 100644 index 000000000..fe1bc5fa2 --- /dev/null +++ b/db/migrate/20221030193943_cleanup_duplicate_pods.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class CleanupDuplicatePods < ActiveRecord::Migration[6.1] + class Pod < ApplicationRecord + end + + def change + reversible do |change| + change.up do + remove_duplicates + + Pod.where(port: nil).update_all(port: -1) # rubocop:disable Rails/SkipsModelValidations + end + + change.down do + Pod.where(port: -1).update_all(port: nil) # rubocop:disable Rails/SkipsModelValidations + end + end + + change_column_null :pods, :port, false + end + + private + + def remove_duplicates + Pod.where(port: nil).group(:host).having("count(*) > 1").pluck(:host).each do |host| + duplicate_ids = Pod.where(host: host).order(:id).ids + target_pod_id = duplicate_ids.shift + + duplicate_ids.each do |pod_id| + Person.where(pod_id: pod_id).update_all(pod_id: target_pod_id) # rubocop:disable Rails/SkipsModelValidations + Pod.delete(pod_id) + end + end + end +end diff --git a/spec/factories.rb b/spec/factories.rb index 04da06dbc..6d0e8de47 100644 --- a/spec/factories.rb +++ b/spec/factories.rb @@ -249,6 +249,7 @@ FactoryBot.define do factory :pod do sequence(:host) {|n| "pod#{n}.example#{r_str}.com" } + port { -1 } ssl { true } end diff --git a/spec/javascripts/app/views/pod_entry_view_spec.js b/spec/javascripts/app/views/pod_entry_view_spec.js index 2c04effc8..c7b6f8968 100644 --- a/spec/javascripts/app/views/pod_entry_view_spec.js +++ b/spec/javascripts/app/views/pod_entry_view_spec.js @@ -30,7 +30,8 @@ describe("app.views.PodEntry", function() { this.pod.set({ status: "no_errors", ssl: true, - host: "pod.example.com" + host: "pod.example.com", + port: -1 }); var actual = this.view.presenter(); expect(actual).toEqual(jasmine.objectContaining({ diff --git a/spec/models/pod_spec.rb b/spec/models/pod_spec.rb index ac5520c73..4c41f0173 100644 --- a/spec/models/pod_spec.rb +++ b/spec/models/pod_spec.rb @@ -16,26 +16,31 @@ describe Pod, type: :model do it "ignores default ports" do pod = Pod.find_or_create_by(url: "https://example.org:443/") expect(pod.host).to eq("example.org") - expect(pod.port).to be_nil + expect(pod.port).to eq(Pod::DEFAULT_PORT) end it "sets ssl boolean" do pod = Pod.find_or_create_by(url: "https://example.org/") expect(pod.ssl).to be true + expect(pod.port).to eq(Pod::DEFAULT_PORT) end it "updates ssl boolean if upgraded to https" do pod = Pod.find_or_create_by(url: "http://example.org/") expect(pod.ssl).to be false + expect(pod.port).to eq(Pod::DEFAULT_PORT) pod = Pod.find_or_create_by(url: "https://example.org/") expect(pod.ssl).to be true + expect(pod.port).to eq(Pod::DEFAULT_PORT) end it "does not update ssl boolean if downgraded to http" do pod = Pod.find_or_create_by(url: "https://example.org/") expect(pod.ssl).to be true + expect(pod.port).to eq(Pod::DEFAULT_PORT) pod = Pod.find_or_create_by(url: "http://example.org/") expect(pod.ssl).to be true + expect(pod.port).to eq(Pod::DEFAULT_PORT) end context "validation" do @@ -205,6 +210,11 @@ describe Pod, type: :model do pod = FactoryBot.create(:pod) expect(pod.url_to("/receive/public")).to eq("https://#{pod.host}/receive/public") end + + it "includes non-default port in pod url" do + pod = FactoryBot.create(:pod, port: 3000) + expect(pod.url_to("/receive/public")).to eq("https://#{pod.host}:#{pod.port}/receive/public") + end end describe "#update_offline_since" do -- cgit v1.2.3 From 8334eeeeff869b2de4457cf5004e490a01542d44 Mon Sep 17 00:00:00 2001 From: Benjamin Neff Date: Mon, 31 Oct 2022 00:43:50 +0100 Subject: Ensure pod urls are always lowercase otherwise pods can exist multiple times with mixed case --- app/models/pod.rb | 4 ++-- .../20221030193943_cleanup_duplicate_pods.rb | 24 ++++++++++++++++------ spec/models/pod_spec.rb | 5 +++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/app/models/pod.rb b/app/models/pod.rb index ff61af5b3..0f3177c2b 100644 --- a/app/models/pod.rb +++ b/app/models/pod.rb @@ -58,7 +58,7 @@ class Pod < ApplicationRecord def find_or_create_by(opts) # Rename this method to not override an AR method uri = URI.parse(opts.fetch(:url)) port = DEFAULT_PORTS.include?(uri.port) ? DEFAULT_PORT : uri.port - find_or_initialize_by(host: uri.host, port: port).tap do |pod| + find_or_initialize_by(host: uri.host.downcase, port: port).tap do |pod| pod.ssl ||= (uri.scheme == "https") pod.save end @@ -168,6 +168,6 @@ class Pod < ApplicationRecord def not_own_pod pod_uri = AppConfig.pod_uri pod_port = DEFAULT_PORTS.include?(pod_uri.port) ? DEFAULT_PORT : pod_uri.port - errors.add(:base, "own pod not allowed") if pod_uri.host == host && pod_port == port + errors.add(:base, "own pod not allowed") if pod_uri.host.downcase == host && pod_port == port end end diff --git a/db/migrate/20221030193943_cleanup_duplicate_pods.rb b/db/migrate/20221030193943_cleanup_duplicate_pods.rb index fe1bc5fa2..18fd5740d 100644 --- a/db/migrate/20221030193943_cleanup_duplicate_pods.rb +++ b/db/migrate/20221030193943_cleanup_duplicate_pods.rb @@ -8,6 +8,7 @@ class CleanupDuplicatePods < ActiveRecord::Migration[6.1] reversible do |change| change.up do remove_duplicates + cleanup_mixed_case_pods Pod.where(port: nil).update_all(port: -1) # rubocop:disable Rails/SkipsModelValidations end @@ -24,13 +25,24 @@ class CleanupDuplicatePods < ActiveRecord::Migration[6.1] def remove_duplicates Pod.where(port: nil).group(:host).having("count(*) > 1").pluck(:host).each do |host| - duplicate_ids = Pod.where(host: host).order(:id).ids - target_pod_id = duplicate_ids.shift + cleanup_duplicates(Pod.where(host: host).order(:id).ids) + end + end - duplicate_ids.each do |pod_id| - Person.where(pod_id: pod_id).update_all(pod_id: target_pod_id) # rubocop:disable Rails/SkipsModelValidations - Pod.delete(pod_id) - end + def cleanup_mixed_case_pods + Pod.where("lower(host) != host").pluck(:host, :port).each do |host, port| + pod_ids = Pod.where("lower(host) = ?", host.downcase).where(port: port).order(:id).ids + cleanup_duplicates(pod_ids.dup) if pod_ids.size > 1 + Pod.find(pod_ids.first).update(host: host.downcase) + end + end + + def cleanup_duplicates(duplicate_ids) + target_pod_id = duplicate_ids.shift + + duplicate_ids.each do |pod_id| + Person.where(pod_id: pod_id).update_all(pod_id: target_pod_id) # rubocop:disable Rails/SkipsModelValidations + Pod.delete(pod_id) end end end diff --git a/spec/models/pod_spec.rb b/spec/models/pod_spec.rb index 4c41f0173..2e3bc9b09 100644 --- a/spec/models/pod_spec.rb +++ b/spec/models/pod_spec.rb @@ -43,6 +43,11 @@ describe Pod, type: :model do expect(pod.port).to eq(Pod::DEFAULT_PORT) end + it "normalizes hostname to lowercase" do + pod = Pod.find_or_create_by(url: "https://eXaMpLe.oRg/") + expect(pod.host).to eq("example.org") + end + context "validation" do it "is valid" do pod = Pod.find_or_create_by(url: "https://example.org/") -- cgit v1.2.3 From 800f3948702be34a56ee5395b0ad1fa003e99372 Mon Sep 17 00:00:00 2001 From: Benjamin Neff Date: Mon, 31 Oct 2022 01:08:34 +0100 Subject: Show ports on pods list otherwise pods with different ports (or without port) all look the same, like if they are duplicates --- app/assets/javascripts/app/views/pod_entry_view.js | 1 + app/assets/templates/pod_table_entry_tpl.jst.hbs | 2 +- spec/javascripts/app/views/pod_entry_view_spec.js | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/app/views/pod_entry_view.js b/app/assets/javascripts/app/views/pod_entry_view.js index 5e45267d1..3d3d098ab 100644 --- a/app/assets/javascripts/app/views/pod_entry_view.js +++ b/app/assets/javascripts/app/views/pod_entry_view.js @@ -27,6 +27,7 @@ app.views.PodEntry = app.views.Base.extend({ presenter: function() { return _.extend({}, this.defaultPresenter(), { /* jshint camelcase: false */ + hasPort: (this.model.get("port") >= 0), is_unchecked: (this.model.get("status")==="unchecked"), has_no_errors: (this.model.get("status")==="no_errors"), has_errors: (this.model.get("status")!=="no_errors"), diff --git a/app/assets/templates/pod_table_entry_tpl.jst.hbs b/app/assets/templates/pod_table_entry_tpl.jst.hbs index 3c2bb0605..76940a4a5 100644 --- a/app/assets/templates/pod_table_entry_tpl.jst.hbs +++ b/app/assets/templates/pod_table_entry_tpl.jst.hbs @@ -7,7 +7,7 @@ {{/if}} -{{host}} +{{host}}{{#if hasPort}}:{{port}}{{/if}} diff --git a/spec/javascripts/app/views/pod_entry_view_spec.js b/spec/javascripts/app/views/pod_entry_view_spec.js index c7b6f8968..8840cd022 100644 --- a/spec/javascripts/app/views/pod_entry_view_spec.js +++ b/spec/javascripts/app/views/pod_entry_view_spec.js @@ -36,6 +36,7 @@ describe("app.views.PodEntry", function() { var actual = this.view.presenter(); expect(actual).toEqual(jasmine.objectContaining({ /* jshint camelcase: false */ + hasPort: false, is_unchecked: false, has_no_errors: true, has_errors: false, -- cgit v1.2.3