cbaines pushed a commit to branch master
in repository data-service.
commit bb84e45c42ad5b7597a30f92075fb7fb1484a6f1
Author: Christopher Baines <[email protected]>
AuthorDate: Fri Nov 29 13:49:01 2024 +0000
Handle conflicts in insert-missing-data-and-return-all-ids
---
guix-data-service/model/utils.scm | 53 ++++++++++++++++++---------------------
1 file changed, 25 insertions(+), 28 deletions(-)
diff --git a/guix-data-service/model/utils.scm
b/guix-data-service/model/utils.scm
index 8721c52..3fda78f 100644
--- a/guix-data-service/model/utils.scm
+++ b/guix-data-service/model/utils.scm
@@ -323,7 +323,7 @@ WHERE table_name = $1"
")"))
missing-data)
", ")
- " RETURNING id"))
+ " ON CONFLICT DO NOTHING"))
(define (format-json json)
;; PostgreSQL formats JSON strings differently to guile-json, so use
@@ -355,16 +355,17 @@ WHERE table_name = $1"
(error (simple-format #f "normalise-values: error: ~A\n"
unknown))))
data))
- (let* ((existing-entries
+ (let* ((flattened-deduplicated-data
+ (if sets-of-data?
+ (delete-duplicates*
+ (concatenate data))
+ (if delete-duplicates?
+ (delete-duplicates* data)
+ data)))
+ (existing-entries
(if use-temporary-table?
(let ((temp-table-name
- (string-append "temp_" table-name))
- (data
- (if sets-of-data?
- (delete-duplicates* (concatenate data))
- (if delete-duplicates?
- (delete-duplicates* data)
- data))))
+ (string-append "temp_" table-name)))
;; Create a temporary table to store the data
(exec-query
conn
@@ -379,12 +380,11 @@ WHERE table_name = $1"
"ANALYZE " temp-table-name))
;; Populate the temporary table
- (if (null? data)
- '()
- (with-time-logging (string-append "populating "
temp-table-name)
- (exec-query conn
- (insert-sql data
- #:table-name temp-table-name))))
+ (unless (null? flattened-deduplicated-data)
+ (with-time-logging (string-append "populating "
temp-table-name)
+ (exec-query conn
+ (insert-sql flattened-deduplicated-data
+ #:table-name temp-table-name))))
;; Use the temporary table to find the existing values
(let ((result
(with-time-logging
@@ -400,7 +400,7 @@ WHERE table_name = $1"
result))
;; If not using a temporary table, just do a single SELECT query
- (if (null? data)
+ (if (null? flattened-deduplicated-data)
'()
(fold
(lambda (data-chunk result)
@@ -411,15 +411,10 @@ WHERE table_name = $1"
(string->number (first result)))
#:vhash result))
vlist-null
- (chunk (if sets-of-data?
- (delete-duplicates*
- (concatenate data))
- data)
+ (chunk flattened-deduplicated-data
3000)))))
(missing-entries
- (let loop ((lst (if sets-of-data?
- (concatenate data)
- data))
+ (let loop ((lst flattened-deduplicated-data)
(result '()))
(if (null? lst)
(if delete-duplicates?
@@ -440,10 +435,12 @@ WHERE table_name = $1"
'()
(append-map!
(lambda (missing-entries-chunk)
- (map (lambda (result)
- (string->number (first result)))
- (exec-query conn
- (insert-sql missing-entries-chunk))))
+ (exec-query conn
+ (insert-sql missing-entries-chunk))
+
+ (map (lambda (row)
+ (string->number (first row)))
+ (exec-query conn (select-query missing-entries-chunk))))
(chunk missing-entries 3000))))
(new-entries-lookup-vhash
@@ -474,4 +471,4 @@ WHERE table_name = $1"
(error "missing entry" field-values))))
data))))
(values all-ids
- new-entries)))
+ (delete-duplicates/sort! new-entries <))))
