Merge pull request #131 from etalab/migration/api-entreprise

Migration/api entreprise

Author: skelz0r

.gitignore

@@ -45,10 +45,12 @@ sandbox
 coverage/
 
 app/migration/.pgpassword
+app/migration/.ovh.yml
 app/migration/dumps/*.csv
 app/migration/dumps/*.sql
 
 /config/credentials/staging.key
 
 # Ignore local Redis snapshots
 /dump.rdb
+app/migration/dumps/data.db

Gemfile

@@ -33,6 +33,7 @@ gem 'rails-i18n', '~> 7.0.9'
 gem 'redis', '>= 4.0.1'
 gem 'sentry-ruby'
 gem 'sentry-rails'
+gem 'sqlite3'
 gem 'state_machines-activerecord'
 gem 'stimulus-rails'
 gem 'turbo-rails'

Gemfile.lock

@@ -533,6 +533,9 @@ GEM
     snaky_hash (2.0.1)
       hashie
       version_gem (~> 1.1, >= 1.1.1)
+    sqlite3 (1.7.3-aarch64-linux)
+    sqlite3 (1.7.3-arm64-darwin)
+    sqlite3 (1.7.3-x86_64-linux)
     state_machines (0.6.0)
     state_machines-activemodel (0.9.0)
       activemodel (>= 6.0)
@@ -647,6 +650,7 @@ DEPENDENCIES
   sentry-rails
   sentry-ruby
   simplecov
+  sqlite3
   state_machines-activerecord
   stimulus-rails
   turbo-rails

app/decorators/authorization_request_event_decorator.rb

@@ -57,6 +57,8 @@ def changelog_builder(diffs)
       diffs.map { |attribute, values|
         if attribute == 'scopes'
           build_scopes_change(values)
+        elsif attribute == 'applicant_id'
+          h.content_tag(:li, build_applicant_change(attribute, values))
         else
           h.content_tag(:li, build_attribute_change(attribute, values))
         end
@@ -73,8 +75,8 @@ def changelog_diffs_without_blank_values
   end
 
   def build_scopes_change(values)
-    new_scopes = values[0] - values[1]
-    removed_scopes = values[1] - values[0]
+    new_scopes = values[1] - values[0]
+    removed_scopes = values[0] - values[1]
 
     [
       new_scopes.map do |scope|
@@ -111,6 +113,21 @@ def build_attribute_change_with_values(attribute, values)
     ).html_safe
   end
 
+  def build_applicant_change(attribute, values)
+    from_user, to_user = values.map { |id| User.find_by(id:) }
+
+    if from_user && to_user
+      t(
+        'authorization_request_event.changelog_entry_applicant',
+        attribute: object.authorization_request.class.human_attribute_name(attribute),
+        old_value: from_user.email,
+        new_value: to_user.email,
+      ).html_safe
+    else
+      t('authorization_request_event.changelog_entry_applicant_with_missing_data')
+    end
+  end
+
   def initial_submit?
     object.authorization_request.events.where(name: 'submit').order(created_at: :asc).first == object
   end

app/mailers/application_mailer.rb

@@ -12,11 +12,11 @@ def extract_host
   def current_host
     case Rails.env
     when 'production'
-      'https://v2.datapass.api.gouv.fr'
+      'https://api-entreprise.v2.datapass.api.gouv.fr'
     when 'test', 'development'
       'http://localhost:3000'
     else
-      "https://#{Rails.env}.v2.datapass.api.gouv.fr"
+      "https://#{Rails.env}.api-entreprise.v2.datapass.api.gouv.fr"
     end
   end
 end

app/migration/README.md

@@ -12,6 +12,9 @@
 Executez la commande: `./app/migration/local_import.sh`
 Cette commande est idempotent.
 
+Executez la commande: `./app/migration/local_build_db.sh`.
+Il faut penser à supprimer la db local si besoin
+
 Pour clean la db locale: `./app/migration/clean_local_db.sh`
 
 Il possible d'effectuer des filtrages sur les données importées (via les options
@@ -29,6 +32,16 @@ situé dans le dossier [`app/migration/import/authorization_requests/`](./import
 cela permet de simplifier les traitements. La classe
 `Import::AuthorizationRequests` s'occupe des attributs communs.
 
+Pour l'import final il faure les creds d'OVH S3 dans le fichier `.ovh.yml` sous le format suivant:
+
+```yaml
+---
+OVH_BUCKET: OVH_BUCKET
+OVH_ACCESS_KEY_ID: OVH_ACCESS_KEY_ID
+OVH_SECRET_ACCESS_KEY: OVH_SECRET_ACCESS_KEY
+OVH_REGION: reg
+```
+
 ### 2.1 Ajout d'un nouveau type d'habilitation.
 
 1. Ajouter le mapping dans `Import::AuthorizationRequests#from_target_api_to_type`

app/migration/alter_sequence_ids.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bundle exec rails runner "
+ActiveRecord::Base.connection.execute(\"select setval('authorization_requests_id_seq', 87000, true);\")
+"

app/migration/clean_local_db.sh

@@ -0,0 +1,21 @@
+class ClearBucket
+  def perform
+    s3_client.list_objects(bucket: bucket_name).contents.each do |object|
+      s3_client.delete_object(bucket: bucket_name, key: object.key)
+    end
+  end
+
+  private
+
+  def s3_client
+    @s3_client ||= Aws::S3::Client.new(
+      endpoint: "https://s3.#{Rails.application.credentials.dig(:ovh_region)}.io.cloud.ovh.net/",
+      credentials: Aws::Credentials.new(Rails.application.credentials.dig(:ovh_access_key_id), Rails.application.credentials.dig(:ovh_secret_key)),
+      region: Rails.application.credentials.dig(:ovh_region).downcase,
+    )
+  end
+
+  def bucket_name
+    "datapass-v2-#{Rails.env}"
+  end
+end

app/migration/clear_bucket.rb

@@ -0,0 +1,9 @@
+module LocalDatabaseUtils
+  def database
+    @database ||= SQLite3::Database.new(db_path)
+  end
+
+  def db_path
+    Rails.root.join('app', 'migration', 'dumps', 'data.db')
+  end
+end

app/migration/concerns/local_database_utils.rb

@@ -0,0 +1,17 @@
+module UserLegacyUtils
+  extend ActiveSupport::Concern
+
+  include LocalDatabaseUtils
+
+  def all_legacy_users_id_to_email
+    @all_legacy_users_id_to_email ||= database.execute('select id, email from users').to_h
+  end
+
+  def all_users_email_to_id
+    @all_users_email_to_id ||= User.pluck(:email, :id).to_h
+  end
+
+  def legacy_user_id_to_user_id(legacy_id)
+    all_users_email_to_id[all_legacy_users_id_to_email[legacy_id.to_i]]
+  end
+end

app/migration/concerns/user_legacy_utils.rb

@@ -0,0 +1,79 @@
+class CreateAuthorizationFromSnapshot
+  include LocalDatabaseUtils
+
+  attr_reader :authorization_request, :event_row
+
+  def initialize(authorization_request, event_row)
+    @authorization_request = authorization_request
+    @event_row = event_row
+  end
+
+  def perform
+    snapshot = find_closest_snapshot
+
+    # XXX Cas où il y a eu une révocation récente (dunno why ?)
+    if snapshot.blank?
+      data = authorization_request.data
+    else
+      snapshot_items = find_snapshot_items(snapshot)
+      data = build_data(snapshot_items)
+    end
+
+    authorization = Authorization.create!(
+      request_id: authorization_request.id,
+      applicant: authorization_request.applicant,
+      data: build_data(snapshot_items),
+      created_at: event_datetime,
+    )
+
+    authorization_request.class.documents.each do |document_identifier|
+      storage_file_model = authorization_request.public_send(document_identifier)
+      next if storage_file_model.blank?
+
+      document = authorization.documents.create!(
+        identifier: document_identifier,
+      )
+      document.file.attach(storage_file_model.blob)
+    end
+
+    authorization
+  end
+
+  private
+
+  def find_closest_snapshot
+    if event_datetime.to_date <= initial_creation_date
+      data = database.execute(
+        'select * from snapshots where enrollment_id = ? order by created_at limit 1',
+        authorization_request.id,
+      )
+    else
+      data = database.execute(
+        'select * from snapshots where enrollment_id = ? and date(created_at) = ? order by created_at limit 1',
+        authorization_request.id,
+        event_datetime.to_date.to_s,
+      )
+    end
+
+    return if data.empty?
+
+    JSON.parse(data[0][-1]).to_h
+  end
+
+  def find_snapshot_items(snapshot)
+    database.execute('select * from snapshot_items where snapshot_id = ?', snapshot['id'])
+  end
+
+  # FIXME que 26 raws clairement ça va se faire à la main je pense
+  def build_data(snapshot_items)
+    authorization_request.data
+  end
+
+  def event_datetime
+    @event_datetime ||= DateTime.parse(event_row['created_at'])
+  end
+
+  def initial_creation_date
+    @initial_creation_date ||= Date.new(2024, 2, 21)
+  end
+end