Skip to content

Commit

Permalink
Create missing services for ptv location in BE etc.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Deraen
Deraen committed Nov 13, 2024
1 parent 55d718a commit 0d4a31f
Show file tree
Hide file tree
Showing 10 changed files with 211 additions and 82 deletions.
11 changes: 8 additions & 3 deletions webapp/dev/user.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@
(assert-running-system)
(:lipas/search (current-system)))

(defn ptv
[]
(assert-running-system)
(:lipas/ptv (current-system)))

(defn reindex-search!
[]
((requiring-resolve 'lipas.backend.search-indexer/main) (db) (search) "search"))
Expand Down Expand Up @@ -68,8 +73,8 @@

(def robot (core/get-user (db) "[email protected]"))

(maintenance/merge-types (db) (search) robot 4530 4510)
(maintenance/merge-types (db) (search) robot 4520 4510)
(maintenance/merge-types (db) (search) robot 4310 4320)
(maintenance/merge-types (db) (search) (ptv) robot 4530 4510)
(maintenance/merge-types (db) (search) (ptv) robot 4520 4510)
(maintenance/merge-types (db) (search) (ptv) robot 4310 4320)

)
90 changes: 50 additions & 40 deletions webapp/src/clj/lipas/backend/core.clj
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,34 @@
(ns lipas.backend.core
(:require
[buddy.hashers :as hashers]
[cheshire.core :as json]
[clojure.core.async :as async]
[clojure.data.csv :as csv]
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[dk.ative.docjure.spreadsheet :as excel]
[lipas.backend.accessibility :as accessibility]
[lipas.backend.analysis.diversity :as diversity]
[lipas.backend.analysis.reachability :as reachability]
[lipas.backend.db.db :as db]
[lipas.backend.elevation :as elevation]
[lipas.backend.email :as email]
[lipas.backend.gis :as gis]
[lipas.backend.jwt :as jwt]
[lipas.backend.newsletter :as newsletter]
[lipas.backend.s3 :as s3]
[lipas.backend.search :as search]
[lipas.data.admins :as admins]
[lipas.data.cities :as cities]
[lipas.data.owners :as owners]
[lipas.data.types :as types]
[lipas.i18n.core :as i18n]
[lipas.integration.utp.cms :as utp-cms]
[lipas.reports :as reports]
[lipas.roles :as roles]
[lipas.utils :as utils]
[taoensso.timbre :as log])
(:import
[java.io OutputStreamWriter]))
(:require [buddy.hashers :as hashers]
[cheshire.core :as json]
[clojure.core.async :as async]
[clojure.data.csv :as csv]
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[dk.ative.docjure.spreadsheet :as excel]
[lipas.backend.accessibility :as accessibility]
[lipas.backend.analysis.diversity :as diversity]
[lipas.backend.analysis.reachability :as reachability]
[lipas.backend.db.db :as db]
[lipas.backend.elevation :as elevation]
[lipas.backend.email :as email]
[lipas.backend.gis :as gis]
[lipas.backend.jwt :as jwt]
[lipas.backend.newsletter :as newsletter]
[lipas.backend.s3 :as s3]
[lipas.backend.search :as search]
[lipas.data.admins :as admins]
[lipas.data.cities :as cities]
[lipas.data.owners :as owners]
[lipas.data.ptv :as ptv-data]
[lipas.data.types :as types]
[lipas.i18n.core :as i18n]
[lipas.integration.utp.cms :as utp-cms]
[lipas.reports :as reports]
[lipas.roles :as roles]
[lipas.utils :as utils]
[taoensso.timbre :as log])
(:import [java.io OutputStreamWriter]))

(def cache "Simple atom cache for things that (hardly) never change."
(atom {}))
Expand Down Expand Up @@ -498,9 +497,9 @@
(defn save-sports-site!
"Saves sports-site to db and search and appends it to outbound
integrations queue."
([db search user sports-site]
(save-sports-site! db search user sports-site false))
([db search user sports-site draft?]
([db search ptv user sports-site]
(save-sports-site! db search ptv user sports-site false))
([db search ptv user sports-site draft?]
(jdbc/with-db-transaction [tx db]
(let [resp (upsert-sports-site! tx user sports-site draft?)
route? (-> resp :type :type-code types/all :geometry-type #{"LineString"})]
Expand All @@ -522,16 +521,27 @@

(add-to-webhook-queue! tx {:lipas-ids [(:lipas-id resp)]}))

;; Sync the updated site to ptv if the ptv integration is enabled
;; Sync the site to PTV if
;; - it was previously sent to PTV (we might archive it now if it no longer looks like PTV candidate)
;; - it is PTV candidate now
;; - do nothing (keep the previous data in PTV if site was previously sent there) if sync-enabled is false
;; Note: if site status or something is updated in Lipas, so that the site is no longer candidate,
;; that doesn't trigger update if sync-enabled is false.
(if (and (not draft?)
(:ptv resp)
(:sync-enabled (:ptv resp)))
(:sync-enabled (:ptv resp))
;; TODO: Check privilage :ptv/basic or such
(or (ptv-data/ptv-candidate? resp)
(ptv-data/is-sent-to-ptv? resp)))
;; TODO: Currently this will create a new sports-site rev.
;; Make it instead update the sports-site already created in the tx?
;; Otherwise each save-sports-site! will create two sports-site revs.
(let [new-ptv-data (:ptv ((resolve 'lipas.backend.ptv.core/upsert-ptv-service-location!)
tx user
{:org (:org-id (:ptv resp))
;; TODO: If this fails, store failure to the site-data so it can be shown on the
;; UI and user can try again. We don't know how often PTV causes problems,
;; and the sports-site save should work even if this fails.
(let [new-ptv-data (:ptv ((resolve 'lipas.backend.ptv.core/sync-ptv!)
tx search ptv user
{:sports-site resp
:org-id (:org-id (:ptv resp))
:lipas-id (:lipas-id resp)
:ptv (:ptv resp)}))]
(log/infof "Sports site updated and PTV integration enabled")
Expand Down
4 changes: 2 additions & 2 deletions webapp/src/clj/lipas/backend/handler.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
exception-handlers)))

(defn create-app
[{:keys [db emailer search mailchimp aws] :as ctx}]
[{:keys [db emailer search mailchimp aws ptv] :as ctx}]
(ring/ring-handler
(ring/router

Expand Down Expand Up @@ -121,7 +121,7 @@
valid? (s/valid? spec body-params)]
(if valid?
{:status 201
:body (core/save-sports-site! db search identity body-params draft?)}
:body (core/save-sports-site! db search ptv identity body-params draft?)}
{:status 400
:body (s/explain-data spec body-params)})))}}]

Expand Down
89 changes: 87 additions & 2 deletions webapp/src/clj/lipas/backend/ptv/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
:service-channel-ids])

(defn upsert-ptv-service-location!
[db ptv-component user {:keys [org lipas-id ptv] :as _m}]
[db ptv-component user {:keys [org-id lipas-id ptv] :as _m}]
(jdbc/with-db-transaction [tx db]
(let [site (db/get-sports-site db lipas-id)
_ (assert (some? site) (str "Sports site " lipas-id " not found in DB"))
Expand All @@ -109,14 +109,17 @@
site (update site :ptv merge ptv)
;; Use the same TS for sourceId, ptv last-sync and site event-date
now (utils/timestamp)
data (ptv-data/->ptv-service-location org gis/wgs84->tm35fin-no-wrap now (core/enrich site))
data (ptv-data/->ptv-service-location org-id gis/wgs84->tm35fin-no-wrap now (core/enrich site))
ptv-resp (if id
(ptv/update-service-location ptv-component id data)
(ptv/create-service-location ptv-component data))
;; Store the new PTV info to Lipas DB
new-ptv-data (-> ptv
(select-keys persisted-ptv-keys)
(assoc :last-sync now
;; Store the current type-code into ptv data, so this can be
;; used to comapre if the services need to recalculated on site data update.
:previous-type-code (:type-code (:type site))
:source-id (:sourceId ptv-resp)
;; Store the PTV status so we can ignore Lipas archived places that we already archived in PTV.
:publishing-status (:publishingStatus ptv-resp)
Expand Down Expand Up @@ -144,6 +147,88 @@
(:value x)))
(:serviceChannelNames ptv-resp))}})))

(comment

(let [ptv-component (:lipas/ptv integrant.repl.state/system)
org-id ptv-data/liminka-org-id-test
services (:itemList (ptv/get-org-services ptv-component org-id))]
(->> services
(utils/index-by :sourceId)
keys)))

;; Used through resolve due to circular dep
;; TODO: Check if code can be moved around to avoid this
^{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn sync-ptv! [tx search ptv-component user {:keys [sports-site ptv org-id lipas-id]}]
(let [type-code (-> sports-site :type :type-code)

previous-sent? (ptv-data/is-sent-to-ptv? sports-site)
candidate-now? (ptv-data/ptv-candidate? sports-site)

to-archive? (and previous-sent?
(not candidate-now?))

;; TODO: to-archive
;; 1. set PTV status Deleted
;; 2. remove :ptv :source-id, :service-channel-ids

type-code-changed? (not= type-code (:previous-type-code ptv))
ptv (if type-code-changed?
(let [types types/all
;; Figure out what services are available in PTV for the site organization
services (:itemList (ptv/get-org-services ptv-component org-id))
source-id->service (->> services
(utils/index-by :sourceId))

;; Check if services for the current/new site type-code exist
missing-services-input [{:service-ids #{}
:sub-category-id (-> sports-site :type :type-code types :sub-category)
:sub-cateogry (-> sports-site :search-meta :type :sub-category :name :fi)}]
missing-services (ptv-data/resolve-missing-services org-id source-id->service nil missing-services-input)

_ (log/infof "Missing services? %s" (pr-str missing-services))

;; TODO: Move the check for missing services to UI, so
;; user can validate the texts.
;; Go through missing services, create data for them and send to PTV
source-id->service (reduce (fn [acc missing]
(let [x (generate-ptv-service-descriptions search
{:sub-category-id (:sub-category-id missing)
:city-codes [(:city-code (:city (:location sports-site)))]})
service (-> missing
(assoc :org-id org-id
:city-codes [(:city-code (:city (:location sports-site)))]
;; :languages ["fi" "se" "en"]
:description (:description x)
:summary (:summary x)))
_ (log/infof "Missing service, generated descriptions: %s" service)

;; Hope this returns data in same format as list services...
resp (upsert-ptv-service! ptv-component service)]
(assoc acc (:source-id service) resp)))
source-id->service
missing-services)

;; Remove old service-ids from :ptv data and add the new.
;; Don't touch other service-ids in the data, those could have be added manually in UI or in PTV.
;; NOTE: OK, PTV updates are likely lost, because our :ptv :service-ids is what the create/update from
;; Lipas previously returned, so if PTV ServiceLocation was modified after that in PTV, we lose those changes.
old-sports-site (assoc-in sports-site [:type :type-code] (:previous-type-code ptv))
old-service-ids (ptv-data/sports-site->service-ids types source-id->service old-sports-site)
new-service-ids (ptv-data/sports-site->service-ids types source-id->service sports-site)]
(log/infof "Site type changed %s => %s, service-ids updated %s => %s"
(:previous-type-code ptv) type-code
old-service-ids new-service-ids)
(update ptv :service-ids (fn [x]
(->> x
(remove (fn [y] (contains? old-service-ids y)))
(into new-service-ids)))))
ptv)
resp (upsert-ptv-service-location! tx ptv-component user {:org-id org-id
:ptv ptv
:lipas-id lipas-id})]
resp))

(defn save-ptv-integration-definitions
"Saves ptv definitions under key :ptv. Does not notify webhooks,
integrations or analysis queues since they're not likely interested
Expand Down
31 changes: 17 additions & 14 deletions webapp/src/clj/lipas/backend/ptv/integration.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,15 +77,17 @@
;; NOTE: deref + swap
(let [x (get @(:tokens ptv) org-id)]
(if (or (not x) (expired? (:payload x)))
(let [new-token (authenticate (merge {:token-url (:token-url ptv)
:username (get-in ptv [:creds :api :username])
:password (get-in ptv [:creds :api :password])
:org-id org-id}
(when (= "test" (:env ptv))
(get-test-credentials org-id))))
(let [token-props (merge {:token-url (:token-url ptv)
:username (get-in ptv [:creds :api :username])
:password (get-in ptv [:creds :api :password])
:org-id org-id}
(when (= "test" (:env ptv))
(get-test-credentials org-id)))
new-token (authenticate token-props)
payload (parse-payload new-token)
x {:token new-token
:payload (parse-payload new-token)}]
(log/infof "Create token %s => %s" org-id new-token)
:payload payload}]
(log/infof "Create token %s => %s (%s)" org-id new-token payload)
(swap! (:tokens ptv) assoc org-id x)
(:token x))
(:token x))))
Expand All @@ -95,7 +97,6 @@
([ptv auth-org-id req retried?]
(let [token (get-token ptv auth-org-id)
req* (-> req
(dissoc :auth-org-id)
(assoc :accept :json
:as :json)
(assoc-in [:headers :Authorization] (str "bearer " token)))
Expand Down Expand Up @@ -157,8 +158,9 @@

(defn create-service
[ptv
{:keys [org-id] :as service}]
(let [params {:url (make-url ptv "/v11/Service")
service]
(let [org-id (:mainResponsibleOrganization service)
params {:url (make-url ptv "/v11/Service")
:method :post
:form-params service}]
(log/infof "Create PTV service %s" service)
Expand All @@ -175,9 +177,10 @@
(defn update-service
[ptv
service-id
{:keys [org-id] :as data}]
data]
(log/info "Update PTV service with id " service-id "and data" data)
(let [params {:url (make-url ptv "/v11/Service/" service-id)
(let [org-id (:mainResponsibleOrganization data)
params {:url (make-url ptv "/v11/Service/" service-id)
:method :put
:form-params data}]
(-> (http ptv org-id params)
Expand All @@ -196,7 +199,7 @@

(defn update-service-location
[ptv service-location-id data]
(let [org-id (-> data :organization :id)
(let [org-id (-> data :organizationId)
params {:url (make-url ptv "/v11/ServiceChannel/ServiceLocation/" service-location-id)
:method :put
:form-params data}]
Expand Down
8 changes: 4 additions & 4 deletions webapp/src/clj/lipas/maintenance.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
[taoensso.timbre :as log]))

(defn merge-types
[db search user type-code-from type-code-to]
[db search ptv user type-code-from type-code-to]
(let [types* (merge old-types/all types/all)]
(assert (contains? types* type-code-from))
(assert (contains? types/all type-code-to))
Expand All @@ -33,11 +33,11 @@
(log/info "Migrating" (count sites) "type" type-code-from " -> " type-code-to)
(doseq [site sites]
(log/info "Migrating" (:lipas-id site))
(core/save-sports-site! db search user (assoc-in site [:type :type-code] type-code-to)))
(core/save-sports-site! db search ptv user (assoc-in site [:type :type-code] type-code-to)))
(log/info "All done!")))

(defn duplicate-point->area-draft
[db search user type-code-from type-code-to]
[db search ptv user type-code-from type-code-to]
(let [types* (merge old-types/all types/all)]
(assert (contains? types* type-code-from))
(assert (contains? types/all type-code-to))
Expand All @@ -47,7 +47,7 @@
(log/info "Migrating" (count sites) "type" type-code-from "(Point) ->" type-code-to "(Polygon)")
(doseq [site sites]
(log/info "Migrating" (:lipas-id site))
(core/save-sports-site! db search user
(core/save-sports-site! db search ptv user
(-> site
(dissoc :lipas-id)
(assoc :event-date (utils/timestamp))
Expand Down
Loading

0 comments on commit 0d4a31f

Please sign in to comment.