This is an automated email from the ASF dual-hosted git repository. damccorm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 68ed7ce48b2 [Playground] Move modify saved snippets functionality to to cloudfunctions (#26026) 68ed7ce48b2 is described below commit 68ed7ce48b2dbe779c2446391aaafd48b8de9b22 Author: Timur Sultanov <timur.sulta...@akvelon.com> AuthorDate: Wed May 3 22:51:25 2023 +0400 [Playground] Move modify saved snippets functionality to to cloudfunctions (#26026) * Move priviledged Datastore operations to Cloudfunctions * Terraform changes to deploy cloud functions * Remove setting for snippets retention period as it's now hardcoded in a cloud function * service account access review * Datastore namespace added * Update .gitignore * Terraform fixes to permissions and namespaces * Move PutSnippet() into a separate CloudFunction * Remove unused role * Do not apply data migrations which are already applied * Move DB migration to external command * Hook up DB migration script to GKE deployment task * Remove trailing whitespace * Apply migrations in ToB integration tests * CloudFunctions review * remove unused code * Fix EOL on EOF * Update Readme * 'Cloud Functions Developer' role added to requirements * Fix issue with namespace value being ignored * Paths for archive_file set relative to path.root * Pass namespace into cloudfunctions * change arcive * fix name * fix module * Cloud Datastore User role added Required to run DB migration tool * Variable renamed --------- Co-authored-by: Sergey Makarkin <ya...@mail.ru> Co-authored-by: Sergey Makarkin <sergey.makar...@akvelon.com> Co-authored-by: Rouslan <115221004+rshamu...@users.noreply.github.com> Co-authored-by: rshamunov <ruslan.shamu...@akvelon.com> --- .gitignore | 1 + learning/tour-of-beam/backend/docker-compose.yml | 1 + playground/backend/README.md | 1 - playground/backend/build.gradle.kts | 4 +- .../backend/cmd/migration_tool/migration_tool.go | 56 ++++++++ .../remove_unused_snippets.go | 5 +- playground/backend/cmd/server/controller_test.go | 21 +-- playground/backend/cmd/server/server.go | 37 ++--- playground/backend/containers/router/Dockerfile | 5 + .../containers/router/docker-compose.local.yml | 1 + playground/backend/containers/router/entrypoint.sh | 15 ++ playground/backend/functions.go | 118 +++++++++++++++ playground/backend/go.mod | 11 +- playground/backend/go.sum | 30 +++- .../backend/internal/db/datastore/datastore_db.go | 136 +++++++++++------- .../internal/db/datastore/datastore_db_test.go | 51 +------ .../internal/db/datastore/emulator_wrapper.go | 2 +- .../migration_base.go} | 28 ++-- .../backend/internal/db/datastore/migration_db.go | 157 ++++++++++++++++++++ playground/backend/internal/db/db.go | 15 +- playground/backend/internal/db/entity/schema.go | 3 +- .../internal/db/mapper/datastore_mapper_test.go | 6 +- .../db/schema/migration/migrations_test.go | 121 ---------------- .../db/schema/{migration => }/migration_v001.go | 36 +++-- .../db/{entity/schema.go => schema/migrations.go} | 9 +- playground/backend/internal/db/schema/version.go | 62 -------- .../backend/internal/environment/application.go | 67 +++++++-- .../internal/environment/environment_service.go | 112 +++++++++------ .../environment/environment_service_test.go | 160 +++++++++++++++++++-- .../backend/internal/environment/property.go | 2 - .../backend/internal/environment/property_test.go | 3 +- .../external_functions_component.go | 127 ++++++++++++++++ playground/backend/internal/tasks/task.go | 6 +- .../backend/internal/utils/datastore_utils.go | 7 +- playground/backend/playground_functions/Dockerfile | 38 +++++ .../schema.go => playground_functions/cmd/main.go} | 21 ++- .../func_enviornment.go} | 23 ++- .../middleware.go} | 17 ++- playground/backend/properties.yaml | 2 - playground/docker-compose.local.yaml | 43 ++++++ playground/index.yaml | 1 - .../templates/deployment-router.yml | 6 + playground/terraform/README.md | 8 +- playground/terraform/build.gradle.kts | 79 +++++++++- .../infrastructure/api_enable/variables.tf | 2 +- .../{ip_address => archive_file}/main.tf | 10 +- .../infrastructure/cloudfunctions/main.tf | 65 +++++++++ .../variables.tf => cloudfunctions/output.tf} | 13 +- .../{api_enable => cloudfunctions}/variables.tf | 24 +++- .../variables.tf => gke_bucket/main.tf} | 18 ++- .../{ip_address/main.tf => gke_bucket/output.tf} | 4 +- .../{ip_address => gke_bucket}/variables.tf | 9 +- .../terraform/infrastructure/ip_address/main.tf | 2 +- .../infrastructure/ip_address/variables.tf | 2 +- playground/terraform/infrastructure/main.tf | 30 +++- playground/terraform/infrastructure/output.tf | 14 +- playground/terraform/infrastructure/setup/iam.tf | 18 ++- .../terraform/infrastructure/setup/output.tf | 6 +- playground/terraform/infrastructure/variables.tf | 6 +- playground/terraform/main.tf | 6 +- playground/terraform/output.tf | 14 +- playground/terraform/variables.tf | 7 +- 62 files changed, 1399 insertions(+), 505 deletions(-) diff --git a/.gitignore b/.gitignore index 5c106839945..5046b64fb4b 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,7 @@ playground/frontend/playground_components_dev/pubspec.lock **/*.tfstate.* **/*.hcl **/*.tfvars +playground/cloudfunction.zip # Ignore Katas auto-generated files **/*-remote-info.yaml diff --git a/learning/tour-of-beam/backend/docker-compose.yml b/learning/tour-of-beam/backend/docker-compose.yml index 50823ab1bfe..d47db7e020b 100644 --- a/learning/tour-of-beam/backend/docker-compose.yml +++ b/learning/tour-of-beam/backend/docker-compose.yml @@ -42,6 +42,7 @@ services: - CACHE_TYPE=local - SDK_CONFIG=/opt/playground/backend/sdks-emulator.yaml - PROTOCOL_TYPE=TCP + - APPLY_MIGRATIONS=True ports: - "8000:8080" depends_on: diff --git a/playground/backend/README.md b/playground/backend/README.md index 3a3759de538..8afb891b548 100644 --- a/playground/backend/README.md +++ b/playground/backend/README.md @@ -129,7 +129,6 @@ These properties are stored in `backend/properties.yaml` file: - `max_snippet_size` - is the file content size limit. Since 1 character occupies 1 byte of memory, and 1 MB is approximately equal to 1000000 bytes, then maximum size of the snippet is 1000000. - `id_length` - is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity. - `removing_unused_snippets_cron` - is the cron expression for the scheduled task to remove unused snippets. -- `removing_unused_snippets_days` - is the number of days after which a snippet becomes unused. ## Running the server app via Docker diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts index ba3c4c5feb4..03060bd61dd 100644 --- a/playground/backend/build.gradle.kts +++ b/playground/backend/build.gradle.kts @@ -72,7 +72,7 @@ task("removeUnusedSnippet") { doLast { exec { executable("go") - args("run", "cmd/remove_unused_snippets.go", "cleanup", + args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "cleanup", "-day_diff", System.getProperty("dayDiff"), "-project_id", System.getProperty("projectId"), "-namespace", System.getProperty("namespace")) } @@ -83,7 +83,7 @@ task("removeSnippet") { doLast { exec { executable("go") - args("run", "cmd/remove_unused_snippets.go", "remove", + args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "remove", "-snippet_id", System.getProperty("snippetId"), "-project_id", System.getProperty("projectId"), "-namespace", System.getProperty("namespace")) } diff --git a/playground/backend/cmd/migration_tool/migration_tool.go b/playground/backend/cmd/migration_tool/migration_tool.go new file mode 100644 index 00000000000..5553f139c78 --- /dev/null +++ b/playground/backend/cmd/migration_tool/migration_tool.go @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "beam.apache.org/playground/backend/internal/constants" + "beam.apache.org/playground/backend/internal/db/datastore" + "beam.apache.org/playground/backend/internal/db/mapper" + "beam.apache.org/playground/backend/internal/db/schema" + "beam.apache.org/playground/backend/internal/logger" + "context" + "flag" + "fmt" + "os" +) + +func main() { + projectId := flag.String("project-id", "", "GCP project id") + sdkConfigPath := flag.String("sdk-config", "", "Path to the sdk config file") + namespace := flag.String("namespace", constants.Namespace, "Datastore namespace") + + flag.Parse() + + ctx := context.WithValue(context.Background(), constants.DatastoreNamespaceKey, *namespace) + + cwd, err := os.Getwd() + if err != nil { + fmt.Printf("Couldn't get the current working directory, err: %s \n", err.Error()) + os.Exit(1) + } + logger.SetupLogger(context.Background(), cwd, *projectId) + + migratedDb, err := datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), nil, *projectId) + if err != nil { + logger.Fatalf("Couldn't create DB client instance, err: %s \n", err.Error()) + os.Exit(1) + } + + if err := migratedDb.ApplyMigrations(ctx, schema.Migrations, *sdkConfigPath); err != nil { + logger.Fatalf("Couldn't apply migrations, err: %s \n", err.Error()) + os.Exit(1) + } +} diff --git a/playground/backend/cmd/remove_unused_snippets.go b/playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go similarity index 96% rename from playground/backend/cmd/remove_unused_snippets.go rename to playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go index 616150feb4a..d5df9fcbfcd 100644 --- a/playground/backend/cmd/remove_unused_snippets.go +++ b/playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go @@ -30,7 +30,7 @@ import ( func createDatastoreClient(ctx context.Context, projectId string) (*datastore.Datastore, error) { pcMapper := mapper.NewPrecompiledObjectMapper() - db, err := datastore.New(ctx, pcMapper, projectId) + db, err := datastore.New(ctx, pcMapper, nil, projectId) if err != nil { logger.Errorf("Couldn't create the database client, err: %s \n", err.Error()) return nil, err @@ -49,7 +49,8 @@ func cleanup(dayDiff int, projectId, namespace string) error { return err } - err = db.DeleteUnusedSnippets(ctx, int32(dayDiff)) + retentionPeriod := time.Duration(dayDiff) * 24 * time.Hour + err = db.DeleteUnusedSnippets(ctx, retentionPeriod) if err != nil { logger.Errorf("Couldn't delete unused code snippets, err: %s \n", err.Error()) return err diff --git a/playground/backend/cmd/server/controller_test.go b/playground/backend/cmd/server/controller_test.go index b4812adba1b..0050fa662e4 100644 --- a/playground/backend/cmd/server/controller_test.go +++ b/playground/backend/cmd/server/controller_test.go @@ -15,6 +15,7 @@ package main import ( + "beam.apache.org/playground/backend/internal/db/schema" "context" "fmt" "io/fs" @@ -42,8 +43,6 @@ import ( datastoreDb "beam.apache.org/playground/backend/internal/db/datastore" "beam.apache.org/playground/backend/internal/db/entity" "beam.apache.org/playground/backend/internal/db/mapper" - "beam.apache.org/playground/backend/internal/db/schema" - "beam.apache.org/playground/backend/internal/db/schema/migration" "beam.apache.org/playground/backend/internal/environment" "beam.apache.org/playground/backend/internal/logger" "beam.apache.org/playground/backend/internal/tests/test_cleaner" @@ -116,7 +115,9 @@ func setupServer(sdk pb.Sdk) *grpc.Server { if err = os.Setenv("APP_WORK_DIR", path); err != nil { panic(err) } - if err = os.Setenv("SDK_CONFIG", "../../../sdks-emulator.yaml"); err != nil { + + sdkConfigPath := "../../../sdks-emulator.yaml" + if err = os.Setenv("SDK_CONFIG", sdkConfigPath); err != nil { panic(err) } if err = os.Setenv("PROPERTY_PATH", "../../."); err != nil { @@ -145,17 +146,17 @@ func setupServer(sdk pb.Sdk) *grpc.Server { panic(err) } - // setup initial data - versions := []schema.Version{ - new(migration.InitialStructure), - new(migration.AddingComplexityProperty), + err = dbEmulator.ApplyMigrations(ctx, schema.Migrations, sdkConfigPath) + if err != nil { + panic(err) } - dbSchema := schema.New(ctx, dbEmulator, appEnv, props, versions) - actualSchemaVersion, err := dbSchema.InitiateData() + + migrationVersion, err := dbEmulator.GetCurrentDbMigrationVersion(ctx) if err != nil { panic(err) } - appEnv.SetSchemaVersion(actualSchemaVersion) + + appEnv.SetSchemaVersion(migrationVersion) // download test data to the Datastore Emulator test_data.DownloadCatalogsWithMockData(ctx) diff --git a/playground/backend/cmd/server/server.go b/playground/backend/cmd/server/server.go index 9f45a177186..cdac6a440f7 100644 --- a/playground/backend/cmd/server/server.go +++ b/playground/backend/cmd/server/server.go @@ -16,12 +16,12 @@ package main import ( + "beam.apache.org/playground/backend/internal/external_functions" "context" "fmt" - "os" - "github.com/improbable-eng/grpc-web/go/grpcweb" "google.golang.org/grpc" + "os" pb "beam.apache.org/playground/backend/internal/api/v1" "beam.apache.org/playground/backend/internal/cache" @@ -32,8 +32,6 @@ import ( "beam.apache.org/playground/backend/internal/db/datastore" "beam.apache.org/playground/backend/internal/db/entity" "beam.apache.org/playground/backend/internal/db/mapper" - "beam.apache.org/playground/backend/internal/db/schema" - "beam.apache.org/playground/backend/internal/db/schema/migration" "beam.apache.org/playground/backend/internal/environment" "beam.apache.org/playground/backend/internal/logger" "beam.apache.org/playground/backend/internal/tasks" @@ -67,22 +65,27 @@ func runServer() error { // Examples catalog should be retrieved and saved to cache only if the server doesn't suppose to run code, i.e. SDK is unspecified // Database setup only if the server doesn't suppose to run code, i.e. SDK is unspecified if envService.BeamSdkEnvs.ApacheBeamSdk == pb.Sdk_SDK_UNSPECIFIED { + externalFunctions := external_functions.NewExternalFunctionsComponent(envService.ApplicationEnvs) + props, err = environment.NewProperties(envService.ApplicationEnvs.PropertyPath()) if err != nil { return err } - dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), envService.ApplicationEnvs.GoogleProjectId()) + dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), externalFunctions, envService.ApplicationEnvs.GoogleProjectId()) if err != nil { return err } downloadCatalogsToDatastoreEmulator(ctx) - if err = setupDBStructure(ctx, dbClient, &envService.ApplicationEnvs, props); err != nil { + migrationVersion, err := dbClient.GetCurrentDbMigrationVersion(ctx) + if err != nil { return err } + envService.ApplicationEnvs.SetSchemaVersion(migrationVersion) + sdks, err := setupSdkCatalog(ctx, cacheService, dbClient) if err != nil { return err @@ -97,7 +100,7 @@ func runServer() error { // Since only router server has the scheduled task, the task creation is here scheduledTasks := tasks.New(ctx) - if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, props.RemovingUnusedSnptsDays, dbClient); err != nil { + if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, externalFunctions); err != nil { return err } } @@ -224,26 +227,6 @@ func setupExamplesCatalogFromDatastore(ctx context.Context, cacheService cache.C return nil } -// setupDBStructure initializes the data structure -func setupDBStructure(ctx context.Context, db db.Database, appEnv *environment.ApplicationEnvs, props *environment.Properties) error { - versions := []schema.Version{ - new(migration.InitialStructure), - new(migration.AddingComplexityProperty), - } - dbSchema := schema.New(ctx, db, appEnv, props, versions) - actualSchemaVersion, err := dbSchema.InitiateData() - if err != nil { - return err - } - if actualSchemaVersion == "" { - errMsg := "schema version must not be empty" - logger.Error(errMsg) - return fmt.Errorf(errMsg) - } - appEnv.SetSchemaVersion(actualSchemaVersion) - return nil -} - func main() { err := runServer() if err != nil { diff --git a/playground/backend/containers/router/Dockerfile b/playground/backend/containers/router/Dockerfile index 75d123189d0..2aab1d579fd 100644 --- a/playground/backend/containers/router/Dockerfile +++ b/playground/backend/containers/router/Dockerfile @@ -39,6 +39,10 @@ RUN go mod tidy -v &&\ cd cmd/server &&\ go build -ldflags="-X main.BuildCommitHash=$GIT_COMMIT -X main.BuildCommitTimestamp=$GIT_TIMESTAMP" -o /go/bin/server_go_backend +# Build migration tool +RUN cd cmd/migration_tool &&\ + go build -o /go/bin/migration_tool + # Null image FROM debian:stable-20221114-slim @@ -61,6 +65,7 @@ ENV PROPERTY_PATH=/opt/playground/backend/ # Copy build result COPY --from=build /go/bin/server_go_backend /opt/playground/backend/ +COPY --from=build /go/bin/migration_tool /opt/playground/backend/ COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/configs/ COPY sdks.yaml /opt/playground/backend/sdks.yaml diff --git a/playground/backend/containers/router/docker-compose.local.yml b/playground/backend/containers/router/docker-compose.local.yml index 2c48784d049..01c7732b8db 100644 --- a/playground/backend/containers/router/docker-compose.local.yml +++ b/playground/backend/containers/router/docker-compose.local.yml @@ -35,6 +35,7 @@ services: CACHE_TYPE: remote CACHE_ADDRESS: redis:6379 SDK_CONFIG: /opt/playground/backend/sdks-emulator.yaml + APPLY_MIGRATIONS: "True" ports: - "8080:8080" depends_on: diff --git a/playground/backend/containers/router/entrypoint.sh b/playground/backend/containers/router/entrypoint.sh index 988302c2a71..ff19c84defa 100755 --- a/playground/backend/containers/router/entrypoint.sh +++ b/playground/backend/containers/router/entrypoint.sh @@ -14,4 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Check if APPLY_MIGRATIONS is set and apply mgirations in such case +if [ -n "$APPLY_MIGRATIONS" ]; then + echo "Applying migrations" + # If SDK_CONFIG is not set, set it to default value + if [ -z "$SDK_CONFIG" ]; then + SDK_CONFIG=/opt/playground/backend/sdks.yaml + fi + + if [ -n "$DATASTORE_NAMESPACE" ]; then + /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG -namespace $DATASTORE_NAMESPACE + else + /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG + fi +fi + /opt/playground/backend/server_go_backend diff --git a/playground/backend/functions.go b/playground/backend/functions.go new file mode 100644 index 00000000000..2d7b2c0ee40 --- /dev/null +++ b/playground/backend/functions.go @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backend + +import ( + "beam.apache.org/playground/backend/internal/constants" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "beam.apache.org/playground/backend/internal/db/datastore" + "beam.apache.org/playground/backend/internal/db/entity" + "beam.apache.org/playground/backend/internal/logger" + "beam.apache.org/playground/backend/playground_functions" + + "beam.apache.org/playground/backend/internal/db/mapper" + "github.com/GoogleCloudPlatform/functions-framework-go/functions" +) + +const retentionPeriod = 100 * time.Hour * 24 + +var db *datastore.Datastore + +func init() { + env := playground_functions.GetEnvironment() + logger.SetupLogger(context.Background(), "", env.GetProjectId()) + + logger.Debugf("Initializing snippets functions\n") + + pcMapper := mapper.NewPrecompiledObjectMapper() + var err error + db, err = datastore.New(context.Background(), pcMapper, nil, env.GetProjectId()) + if err != nil { + fmt.Printf("Couldn't create the database client, err: %s\n", err.Error()) + panic(err) + } + + ensurePost := playground_functions.EnsureMethod(http.MethodPost) + + functions.HTTP("cleanupSnippets", ensurePost(cleanupSnippets)) + functions.HTTP("putSnippet", ensurePost(putSnippet)) + functions.HTTP("incrementSnippetViews", ensurePost(incrementSnippetViews)) +} + +func handleError(w http.ResponseWriter, statusCode int, err error) { + // Return 500 error and error message + w.WriteHeader(statusCode) + _, werr := w.Write([]byte(err.Error())) + if werr != nil { + logger.Errorf("Couldn't write error message, err: %s \n", werr.Error()) + } +} + +// cleanupSnippets removes old snippets from the database. +func cleanupSnippets(w http.ResponseWriter, r *http.Request) { + namespace := r.URL.Query().Get("namespace") + ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace) + + err := db.DeleteUnusedSnippets(ctx, retentionPeriod) + if err != nil { + logger.Errorf("Couldn't delete unused code snippets, err: %s \n", err.Error()) + handleError(w, http.StatusInternalServerError, err) + return + } + + w.WriteHeader(http.StatusOK) +} + +func putSnippet(w http.ResponseWriter, r *http.Request) { + snipId := r.URL.Query().Get("snipId") + namespace := r.URL.Query().Get("namespace") + ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace) + + var snip entity.Snippet + err := json.NewDecoder(r.Body).Decode(&snip) + if err != nil { + logger.Errorf("Couldn't decode request body, err: %s \n", err.Error()) + handleError(w, http.StatusBadRequest, err) + return + } + + err = db.PutSnippetDirect(ctx, snipId, &snip) + if err != nil { + logger.Errorf("Couldn't put snippet to the database, err: %s \n", err.Error()) + handleError(w, http.StatusInternalServerError, err) + return + } + + w.WriteHeader(http.StatusOK) +} + +func incrementSnippetViews(w http.ResponseWriter, r *http.Request) { + snipId := r.URL.Query().Get("snipId") + namespace := r.URL.Query().Get("namespace") + ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace) + + err := db.IncrementSnippetVisitorsCount(ctx, snipId) + if err != nil { + logger.Errorf("Couldn't increment snippet visitors count for snipId %s, err: %s \n", snipId, err.Error()) + handleError(w, http.StatusInternalServerError, err) + return + } +} diff --git a/playground/backend/go.mod b/playground/backend/go.mod index a2580231ff1..d1ba16a2d59 100644 --- a/playground/backend/go.mod +++ b/playground/backend/go.mod @@ -20,9 +20,11 @@ go 1.18 require ( cloud.google.com/go/datastore v1.9.0 cloud.google.com/go/logging v1.5.0 + github.com/GoogleCloudPlatform/functions-framework-go v1.6.1 github.com/confluentinc/confluent-kafka-go v1.9.2 github.com/go-redis/redis/v8 v8.11.5 github.com/go-redis/redismock/v8 v8.0.6 + github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/improbable-eng/grpc-web v0.15.0 github.com/linkedin/goavro v2.1.0+incompatible @@ -30,7 +32,6 @@ require ( github.com/rs/cors v1.8.2 github.com/spf13/viper v1.14.0 github.com/stretchr/testify v1.8.1 - github.com/google/go-cmp v0.5.9 go.uber.org/goleak v1.2.0 google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 @@ -41,8 +42,10 @@ require ( cloud.google.com/go v0.104.0 // indirect cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/functions v1.7.0 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cloudevents/sdk-go/v2 v2.6.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -53,9 +56,12 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.11.7 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -65,6 +71,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.1 // indirect go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.21.0 // indirect golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/playground/backend/go.sum b/playground/backend/go.sum index 09c9b06f440..1223bc87476 100644 --- a/playground/backend/go.sum +++ b/playground/backend/go.sum @@ -53,6 +53,9 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.9.0 h1:s3Gy1QRIwKxcMCCwJJq/4c64VjROZu6tq1DC632hZuo= cloud.google.com/go/datastore v1.9.0/go.mod h1:yKk5PbPPCtuObGXNWvpQGEyWe+kiMQlTnpMjtltPNTc= +cloud.google.com/go/functions v1.0.0/go.mod h1:O9KS8UweFVo6GbbbCBKh5yEzbW08PVkg2spe3RfPMd4= +cloud.google.com/go/functions v1.7.0 h1:s3Snbr2O4j4p7CuwImBas8rNNmkHS1YJANcCpKGqQSE= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/logging v1.5.0 h1:DcR52smaYLgeK9KPzJlBJyyBYqW/EGKiuRRl8boL1s4= cloud.google.com/go/logging v1.5.0/go.mod h1:c/57U/aLdzSFuBtvbtFduG1Ii54uSm95HOBnp58P7/U= @@ -70,6 +73,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/functions-framework-go v1.6.1 h1:xy2RD54qi/vya4c+Jrh/3yS5JLcTpK167AY47AI4Tdc= +github.com/GoogleCloudPlatform/functions-framework-go v1.6.1/go.mod h1:pq+lZy4vONJ5fjd3q/B6QzWhfHPAbuVweLpxZzMOb9Y= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -94,6 +99,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -112,6 +119,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudevents/sdk-go/v2 v2.6.1 h1:yHtzgmeBvc0TZx1nrnvYXov1CSvkQyvhEhNMs8Z5Mmk= +github.com/cloudevents/sdk-go/v2 v2.6.1/go.mod h1:nlXhgFkf0uTopxmRXalyMwS2LG70cRGPrxzmjJgSG0U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -286,6 +295,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -365,6 +375,7 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -423,6 +434,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -434,6 +446,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -472,6 +486,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -568,6 +583,8 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -594,13 +611,22 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1024,6 +1050,7 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1110,8 +1137,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/playground/backend/internal/db/datastore/datastore_db.go b/playground/backend/internal/db/datastore/datastore_db.go index 6f4cbf074e9..c110a61487a 100644 --- a/playground/backend/internal/db/datastore/datastore_db.go +++ b/playground/backend/internal/db/datastore/datastore_db.go @@ -16,6 +16,7 @@ package datastore import ( + "beam.apache.org/playground/backend/internal/external_functions" "context" "fmt" "time" @@ -39,21 +40,26 @@ const ( ) type Datastore struct { - Client *datastore.Client - ResponseMapper mapper.ResponseMapper + Client *datastore.Client + ResponseMapper mapper.ResponseMapper + externalFunctions external_functions.ExternalFunctions } -func New(ctx context.Context, responseMapper mapper.ResponseMapper, projectId string) (*Datastore, error) { +func New(ctx context.Context, responseMapper mapper.ResponseMapper, externalFunctions external_functions.ExternalFunctions, projectId string) (*Datastore, error) { client, err := datastore.NewClient(ctx, projectId) if err != nil { logger.Errorf("Datastore: connection to store: error during connection, err: %s\n", err.Error()) return nil, err } - return &Datastore{Client: client, ResponseMapper: responseMapper}, nil + return &Datastore{ + Client: client, + ResponseMapper: responseMapper, + externalFunctions: externalFunctions, + }, nil } // Delete unused snippets by given persistenceKey -func (d *Datastore) deleteObsoleteSnippets(ctx context.Context, snipKey *datastore.Key, persistenceKey string) error { +func (d *Datastore) DeleteObsoleteSnippets(ctx context.Context, snipKey *datastore.Key, persistenceKey string) error { if persistenceKey == "" || snipKey == nil { logger.Debugf("no persistence key or no current snippet key") return nil @@ -63,19 +69,42 @@ func (d *Datastore) deleteObsoleteSnippets(ctx context.Context, snipKey *datasto Namespace(utils.GetNamespace(ctx)). FilterField("persistenceKey", "=", persistenceKey) - // At the moment, datastore emulator doesn't allow != filters, - // hence this crutches - // https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues - // When it's fixed, post-query filter could be replaced with - // - // FilterField("__key__", "!=", snipKey) + // At the moment, datastore emulator doesn't allow != filters, + // hence this crutches + // https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues + // When it's fixed, post-query filter could be replaced with + // + // FilterField("__key__", "!=", snipKey) return d.deleteSnippets(ctx, snippetQuery, snipKey) } -// PutSnippet puts the snippet entity to datastore +// PutSnippet puts the snippet entity to datastore using cloud function proxy func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip *entity.Snippet) error { logger.Debugf("putting snippet %q, persistent key %q...", snipId, snip.Snippet.PersistenceKey) + + var err error + if d.externalFunctions != nil { + err = d.externalFunctions.PutSnippet(ctx, snipId, snip) + } + if err != nil || d.externalFunctions == nil { + if err != nil { + logger.Errorf("Datastore: PutSnippet(): error during the PutSnippet() call to the cloud function, "+ + "accessing the datastore directly, err: %s\n", err.Error()) + } + if d.externalFunctions == nil { + logger.Warnf("Datastore: PutSnippet(): external functions are not set, " + + "accessing the datastore directly") + } + return d.PutSnippetDirect(ctx, snipId, snip) + } + + return nil +} + +// PutSnippetDirect puts the snippet entity to datastore +func (d *Datastore) PutSnippetDirect(ctx context.Context, snipId string, snip *entity.Snippet) error { + logger.Debugf("putting snippet %q, persistent key %q...", snipId, snip.Snippet.PersistenceKey) if snip == nil { logger.Errorf("Datastore: PutSnippet(): snippet is nil") return nil @@ -105,7 +134,8 @@ func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip *entity. return err } - return d.deleteObsoleteSnippets(ctx, snipKey, snip.Snippet.PersistenceKey) + // Delete the previous version of the snippet + return d.DeleteObsoleteSnippets(ctx, snipKey, snip.Snippet.PersistenceKey) } // GetSnippet returns the snippet entity by identifier @@ -113,53 +143,56 @@ func (d *Datastore) GetSnippet(ctx context.Context, id string) (*entity.SnippetE key := utils.GetSnippetKey(ctx, id) snip := new(entity.SnippetEntity) + err := d.Client.Get(ctx, key, snip) + if err != nil { + logger.Errorf("Datastore: GetSnippet(): error during snippet getting, err: %s\n", err.Error()) + return nil, err + } + + logger.Infof("Datastore: GetSnippet(): snippet %s has %d view count", id, snip.VisitCount) + + // Update the last visited time and visit count if possible + err = nil + if d.externalFunctions != nil { + err = d.externalFunctions.IncrementSnippetViews(ctx, id) + } + if err != nil || d.externalFunctions == nil { + if err != nil { + logger.Errorf("Datastore: GetSnippet(): error during updating snippet visit count using"+ + " cloud function proxy, err: %s\n", err.Error()) + } + if d.externalFunctions == nil { + logger.Warnf("Datastore: GetSnippet(): cloud function proxy is not initialized, " + + "trying to call IncrementSnippetVisitorsCount() directly.") + } + err = d.IncrementSnippetVisitorsCount(ctx, id) + if err != nil { + logger.Errorf("Datastore: GetSnippet(): Can't increment snippet visit count, skipping view increment") + } + } + + return snip, nil +} + +func (d *Datastore) IncrementSnippetVisitorsCount(ctx context.Context, id string) error { + key := utils.GetSnippetKey(ctx, id) + snip := new(entity.SnippetEntity) + _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { if err := tx.Get(key, snip); err != nil { - logger.Errorf("Datastore: GetSnippet(): error during snippet getting, err: %s\n", err.Error()) + logger.Errorf("Datastore: IncrementSnippetVisitorsCount(): error during snippet getting, err: %s\n", err.Error()) return err } snip.LVisited = time.Now() snip.VisitCount += 1 if _, err := tx.Put(key, snip); err != nil { - logger.Errorf("Datastore: GetSnippet(): error during snippet setting, err: %s\n", err.Error()) + logger.Errorf("Datastore: IncrementSnippetVisitorsCount(): error during snippet setting, err: %s\n", err.Error()) return err } return nil }) if err != nil { - logger.Errorf("Datastore: GetSnippet: error updating snippet: %s\n", err.Error()) - return nil, err - } - - return snip, nil -} - -// PutSchemaVersion puts the schema entity to datastore -func (d *Datastore) PutSchemaVersion(ctx context.Context, id string, schema *entity.SchemaEntity) error { - if schema == nil { - logger.Errorf("Datastore: PutSchemaVersion(): schema version is nil") - return nil - } - key := utils.GetSchemaVerKey(ctx, id) - if _, err := d.Client.Put(ctx, key, schema); err != nil { - logger.Errorf("Datastore: PutSchemaVersion(): error during entity saving, err: %s\n", err.Error()) - return err - } - return nil -} - -// PutSDKs puts the SDK entity to datastore -func (d *Datastore) PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error { - if sdks == nil || len(sdks) == 0 { - logger.Errorf("Datastore: PutSDKs(): sdks are empty") - return nil - } - var keys []*datastore.Key - for _, sdk := range sdks { - keys = append(keys, utils.GetSdkKey(ctx, sdk.Name)) - } - if _, err := d.Client.PutMulti(ctx, keys, sdks); err != nil { - logger.Errorf("Datastore: PutSDK(): error during entity saving, err: %s\n", err.Error()) + logger.Errorf("Datastore: IncrementSnippetVisitorsCount: error updating snippet: %s\n", err.Error()) return err } return nil @@ -536,10 +569,9 @@ func (d *Datastore) DeleteSnippet(ctx context.Context, id string) error { return d.deleteSnippetByKey(ctx, key) } -// DeleteUnusedSnippets deletes all unused snippets -func (d *Datastore) DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error { - var hoursDiff = dayDiff * 24 - boundaryDate := time.Now().Add(-time.Hour * time.Duration(hoursDiff)) +// DeleteUnusedSnippets deletes all unused snippets older than retentionPeriod +func (d *Datastore) DeleteUnusedSnippets(ctx context.Context, retentionPeriod time.Duration) error { + boundaryDate := time.Now().Add(-retentionPeriod) snippetQuery := datastore.NewQuery(constants.SnippetKind). Namespace(utils.GetNamespace(ctx)). FilterField("lVisited", "<=", boundaryDate). diff --git a/playground/backend/internal/db/datastore/datastore_db_test.go b/playground/backend/internal/db/datastore/datastore_db_test.go index 835ee90e163..61d8e35049c 100644 --- a/playground/backend/internal/db/datastore/datastore_db_test.go +++ b/playground/backend/internal/db/datastore/datastore_db_test.go @@ -249,53 +249,6 @@ func TestDatastore_PutSDKs(t *testing.T) { } } -func TestDatastore_PutSchemaVersion(t *testing.T) { - type args struct { - ctx context.Context - id string - schema *entity.SchemaEntity - } - tests := []struct { - name string - args args - wantErr bool - cleanData func() - }{ - { - name: "PutSchemaVersion() in the usual case", - args: args{ - ctx: ctx, - id: "MOCK_ID", - schema: &entity.SchemaEntity{Descr: "MOCK_DESCRIPTION"}, - }, - wantErr: false, - cleanData: func() { - test_cleaner.CleanSchemaVersion(ctx, t, "MOCK_ID") - }, - }, - { - name: "PutSchemaVersion() when input data is nil", - args: args{ - ctx: ctx, - id: "MOCK_ID", - schema: nil, - }, - wantErr: false, - cleanData: func() {}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := datastoreDb.PutSchemaVersion(tt.args.ctx, tt.args.id, tt.args.schema) - if err != nil { - t.Error("PutSchemaVersion() method failed") - } - tt.cleanData() - }) - } -} - func TestDatastore_GetFiles(t *testing.T) { type args struct { ctx context.Context @@ -858,7 +811,7 @@ func TestDatastore_DeleteUnusedSnippets(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.prepare() - err := datastoreDb.DeleteUnusedSnippets(tt.args.ctx, tt.args.dayDiff) + err := datastoreDb.DeleteUnusedSnippets(tt.args.ctx, time.Duration(tt.args.dayDiff)*time.Hour*24) if (err != nil) != tt.wantErr { t.Errorf("DeleteUnusedSnippets() error = %v, wantErr %v", err, tt.wantErr) } @@ -939,7 +892,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := New(ctx, mapper.NewPrecompiledObjectMapper(), constants.EmulatorProjectId) + _, err := New(ctx, mapper.NewPrecompiledObjectMapper(), nil, constants.EmulatorProjectId) if (err != nil) != tt.wantErr { t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/playground/backend/internal/db/datastore/emulator_wrapper.go b/playground/backend/internal/db/datastore/emulator_wrapper.go index 2aff281f2a0..9252c454b01 100644 --- a/playground/backend/internal/db/datastore/emulator_wrapper.go +++ b/playground/backend/internal/db/datastore/emulator_wrapper.go @@ -81,7 +81,7 @@ func NewEmulated(ctx context.Context) (*EmulatedDatastore, error) { panic(err) } - datastoreDb, err := New(ctx, mapper.NewPrecompiledObjectMapper(), constants.EmulatorProjectId) + datastoreDb, err := New(ctx, mapper.NewPrecompiledObjectMapper(), nil, constants.EmulatorProjectId) if err != nil { return nil, err } diff --git a/playground/backend/internal/db/schema/migration/migration_v002.go b/playground/backend/internal/db/datastore/migration_base.go similarity index 56% rename from playground/backend/internal/db/schema/migration/migration_v002.go rename to playground/backend/internal/db/datastore/migration_base.go index 4d2ed04c7ae..f58b0a5d2d2 100644 --- a/playground/backend/internal/db/schema/migration/migration_v002.go +++ b/playground/backend/internal/db/datastore/migration_base.go @@ -13,28 +13,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package migration +package datastore import ( - "beam.apache.org/playground/backend/internal/db/entity" - "beam.apache.org/playground/backend/internal/db/schema" + "cloud.google.com/go/datastore" + "context" ) -type AddingComplexityProperty struct { +type Migration interface { + Apply(ctx context.Context, tx *datastore.Transaction, sdkConfigPath string) error + GetVersion() int + GetDescription() string } -func (is *AddingComplexityProperty) InitiateData(args *schema.DBArgs) error { - schemaEntity := &entity.SchemaEntity{Descr: is.GetDescription()} - if err := args.Db.PutSchemaVersion(args.Ctx, is.GetVersion(), schemaEntity); err != nil { - return err - } - return nil +type MigrationBase struct { + Version int + Description string } -func (is *AddingComplexityProperty) GetVersion() string { - return "0.0.2" +func (m MigrationBase) GetVersion() int { + return m.Version } -func (is AddingComplexityProperty) GetDescription() string { - return "Adding a complexity property to the example entity" +func (m MigrationBase) GetDescription() string { + return m.Description } diff --git a/playground/backend/internal/db/datastore/migration_db.go b/playground/backend/internal/db/datastore/migration_db.go new file mode 100644 index 00000000000..72946bedc0c --- /dev/null +++ b/playground/backend/internal/db/datastore/migration_db.go @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "beam.apache.org/playground/backend/internal/constants" + "beam.apache.org/playground/backend/internal/db/entity" + "beam.apache.org/playground/backend/internal/logger" + "beam.apache.org/playground/backend/internal/utils" + "cloud.google.com/go/datastore" + "context" + "errors" +) + +// GetCurrentDbMigrationVersion returns the current version of the schema +func (d *Datastore) GetCurrentDbMigrationVersion(ctx context.Context) (int, error) { + query := datastore.NewQuery(constants.SchemaKind). + Namespace(utils.GetNamespace(ctx)).Order("-version").Limit(1) + var schemas []*entity.SchemaEntity + if _, err := d.Client.GetAll(ctx, query, &schemas); err != nil { + logger.Errorf("Datastore: GetCurrentDbMigrationVersion(): error during getting current version, err: %s\n", err.Error()) + return -1, err + } + if len(schemas) == 0 { + logger.Errorf("Datastore: GetCurrentDbMigrationVersion(): no schema versions found\n") + return -1, errors.New("no schema versions found") + } + return schemas[0].Version, nil +} + +// HasSchemaVersion returns true if the schema version is applied +func (d *Datastore) hasSchemaVersion(ctx context.Context, version int) (bool, error) { + key := utils.GetSchemaVerKey(ctx, version) + schemaEntity := new(entity.SchemaEntity) + err := d.Client.Get(ctx, key, schemaEntity) + if err != nil { + if err == datastore.ErrNoSuchEntity { + return false, nil + } + logger.Errorf("Datastore: hasSchemaVersion(): error during getting schema version, err: %s\n", err.Error()) + return false, err + } + logger.Infof("Datastore: hasSchemaVersion(): found SchemaEntity: %v\n", schemaEntity) + return true, nil +} + +// applyMigration applies the given migration to the database. +func (d *Datastore) applyMigration(ctx context.Context, migration Migration, sdkConfigPath string) error { + logger.Infof("Datastore: applyMigration(): applying migration \"%d: %s\"\n", migration.GetVersion(), migration.GetDescription()) + + _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { + if err := migration.Apply(ctx, tx, sdkConfigPath); err != nil { + logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, rolling back, err: %s\n", + migration.GetVersion(), + migration.GetDescription(), + err.Error()) + return err + } + + // Record the migration version + if err := putSchemaVersion(ctx, tx, migration.GetVersion(), migration.GetDescription()); err != nil { + logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, rolling back, err: %s\n", + migration.GetVersion(), + migration.GetDescription(), + err.Error()) + return err + } + + return nil + }) + + if err != nil { + logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, err: %s\n", + migration.GetVersion(), + migration.GetDescription(), + err.Error()) + return err + } + + logger.Infof("Datastore: applyMigration(): migration \"%d: %s\" applied successfully\n", migration.GetVersion(), migration.GetDescription()) + return nil +} + +// ApplyMigrations applies all migrations to the database. +func (d *Datastore) ApplyMigrations(ctx context.Context, migrations []Migration, sdkConfigPath string) error { + for _, migration := range migrations { + if applied, err := d.hasSchemaVersion(ctx, migration.GetVersion()); err != nil { + logger.Errorf("Datastore: ApplyMigrations(): Error checking migration \"%d: %s\" : %s", migration.GetVersion(), migration.GetDescription(), err.Error()) + return err + } else if applied { + logger.Infof("Datastore: ApplyMigrations(): migration \"%d: %s\" already applied, skipping\n", migration.GetVersion(), migration.GetDescription()) + continue + } + + if err := d.applyMigration(ctx, migration, sdkConfigPath); err != nil { + logger.Errorf("Datastore: ApplyMigrations(): Error applying migration \"%d: %s\" : %s", migration.GetVersion(), migration.GetDescription(), err.Error()) + return err + } + } + return nil +} + +// putSchemaVersion puts the schema entity to datastore +func putSchemaVersion(ctx context.Context, tx *datastore.Transaction, version int, description string) error { + key := utils.GetSchemaVerKey(ctx, version) + if _, err := tx.Put(key, &entity.SchemaEntity{ + Version: version, + Descr: description, + }); err != nil { + logger.Errorf("Datastore: putSchemaVersion(): error during entity saving, err: %s\n", err.Error()) + return err + } + return nil +} + +func (d *Datastore) PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error { + _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { + return TxPutSDKs(ctx, tx, sdks) + }) + + if err != nil { + logger.Errorf("Datastore: PutSDKs(): error during transaction, err: %s\n", err.Error()) + return err + } + + return nil +} + +// TxPutSDKs puts the SDK entity to datastore in a transaction +func TxPutSDKs(ctx context.Context, tx *datastore.Transaction, sdks []*entity.SDKEntity) error { + if sdks == nil || len(sdks) == 0 { + logger.Errorf("Datastore: TxPutSDKs(): sdks are empty") + return nil + } + var keys []*datastore.Key + for _, sdk := range sdks { + keys = append(keys, utils.GetSdkKey(ctx, sdk.Name)) + } + if _, err := tx.PutMulti(keys, sdks); err != nil { + logger.Errorf("Datastore: TxPutSDKs(): error during entity saving, err: %s\n", err.Error()) + return err + } + return nil +} diff --git a/playground/backend/internal/db/db.go b/playground/backend/internal/db/db.go index 8f73a57a59a..ff1cb5154e8 100644 --- a/playground/backend/internal/db/db.go +++ b/playground/backend/internal/db/db.go @@ -16,7 +16,9 @@ package db import ( + "beam.apache.org/playground/backend/internal/db/datastore" "context" + "time" pb "beam.apache.org/playground/backend/internal/api/v1" "beam.apache.org/playground/backend/internal/db/entity" @@ -26,6 +28,7 @@ type Database interface { SnippetDatabase CatalogDatabase ExampleDatabase + MigrationsDatabase } type SnippetDatabase interface { @@ -35,14 +38,10 @@ type SnippetDatabase interface { GetFiles(ctx context.Context, snipId string, numberOfFiles int) ([]*entity.FileEntity, error) - DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error + DeleteUnusedSnippets(ctx context.Context, retentionPeriod time.Duration) error } type CatalogDatabase interface { - PutSchemaVersion(ctx context.Context, id string, schema *entity.SchemaEntity) error - - PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error - GetSDKs(ctx context.Context) ([]*entity.SDKEntity, error) } @@ -61,3 +60,9 @@ type ExampleDatabase interface { GetExampleGraph(ctx context.Context, id string) (string, error) } + +type MigrationsDatabase interface { + GetCurrentDbMigrationVersion(ctx context.Context) (int, error) + + ApplyMigrations(ctx context.Context, migrations []datastore.Migration, sdkConfigPath string) error +} diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/internal/db/entity/schema.go index 43aaacfca34..a0fd0e72e3e 100644 --- a/playground/backend/internal/db/entity/schema.go +++ b/playground/backend/internal/db/entity/schema.go @@ -16,5 +16,6 @@ package entity type SchemaEntity struct { - Descr string `datastore:"descr,noindex"` + Version int `datastore:"version"` + Descr string `datastore:"descr,noindex"` } diff --git a/playground/backend/internal/db/mapper/datastore_mapper_test.go b/playground/backend/internal/db/mapper/datastore_mapper_test.go index b84694824d2..3acb44a9530 100644 --- a/playground/backend/internal/db/mapper/datastore_mapper_test.go +++ b/playground/backend/internal/db/mapper/datastore_mapper_test.go @@ -31,8 +31,8 @@ var testable *DatastoreMapper var datastoreMapperCtx = context.Background() func TestMain(m *testing.M) { - appEnv := environment.NewApplicationEnvs("/app", "", "", "", "", "../../../.", "", "", nil, 0, 0) - appEnv.SetSchemaVersion("MOCK_SCHEMA") + appEnv := environment.NewApplicationEnvs("/app", "", "", "", "", "../../../.", "", "", "", "", "", nil, 0, 0) + appEnv.SetSchemaVersion(1) props, _ := environment.NewProperties(appEnv.PropertyPath()) testable = NewDatastoreMapper(datastoreMapperCtx, appEnv, props) exitValue := m.Run() @@ -59,7 +59,7 @@ func TestEntityMapper_ToSnippet(t *testing.T) { IdLength: 11, }, Snippet: &entity.SnippetEntity{ - SchVer: utils.GetSchemaVerKey(datastoreMapperCtx, "MOCK_SCHEMA"), + SchVer: utils.GetSchemaVerKey(datastoreMapperCtx, 1), Sdk: utils.GetSdkKey(datastoreMapperCtx, pb.Sdk_SDK_JAVA.String()), PipeOpts: "MOCK_OPTIONS", Origin: constants.UserSnippetOrigin, diff --git a/playground/backend/internal/db/schema/migration/migrations_test.go b/playground/backend/internal/db/schema/migration/migrations_test.go deleted file mode 100644 index 42218e9f4ac..00000000000 --- a/playground/backend/internal/db/schema/migration/migrations_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "context" - "os" - "testing" - - "beam.apache.org/playground/backend/internal/constants" - "beam.apache.org/playground/backend/internal/db/datastore" - "beam.apache.org/playground/backend/internal/db/schema" - "beam.apache.org/playground/backend/internal/environment" -) - -var datastoreDb *datastore.EmulatedDatastore -var ctx context.Context -var appEnvs *environment.ApplicationEnvs -var props *environment.Properties - -func TestMain(m *testing.M) { - setup() - code := m.Run() - teardown() - os.Exit(code) -} - -func setup() { - ctx = context.Background() - ctx = context.WithValue(ctx, constants.DatastoreNamespaceKey, "migration") - var err error - datastoreDb, err = datastore.NewEmulated(ctx) - if err != nil { - panic(err) - } - if err != nil { - panic(err) - } - appEnvs = environment.NewApplicationEnvs("/app", "", "", "", "../../../../../sdks-emulator.yaml", "../../../../.", "", "", nil, 0, 0) - props, err = environment.NewProperties(appEnvs.PropertyPath()) - if err != nil { - panic(err) - } -} - -func teardown() { - clientCloseErr := datastoreDb.Close() - if clientCloseErr != nil { - panic(clientCloseErr) - } -} - -func TestInitialStructure_InitiateData(t *testing.T) { - tests := []struct { - name string - dbArgs *schema.DBArgs - wantErr bool - }{ - { - name: "Test migration with version 0.0.1 in the usual case", - dbArgs: &schema.DBArgs{ - Ctx: ctx, - Db: datastoreDb, - AppEnv: appEnvs, - Props: props, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - is := new(InitialStructure) - err := is.InitiateData(tt.dbArgs) - if (err != nil) != tt.wantErr { - t.Errorf("InitiateData(): error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestAddingComplexityProperty_InitiateData(t *testing.T) { - tests := []struct { - name string - dbArgs *schema.DBArgs - wantErr bool - }{ - { - name: "Test migration with version 0.0.2 in the usual case", - dbArgs: &schema.DBArgs{ - Ctx: ctx, - Db: datastoreDb, - AppEnv: appEnvs, - Props: props, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - is := new(AddingComplexityProperty) - err := is.InitiateData(tt.dbArgs) - if (err != nil) != tt.wantErr { - t.Errorf("InitiateData(): error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - -} diff --git a/playground/backend/internal/db/schema/migration/migration_v001.go b/playground/backend/internal/db/schema/migration_v001.go similarity index 74% rename from playground/backend/internal/db/schema/migration/migration_v001.go rename to playground/backend/internal/db/schema/migration_v001.go index 38550859c65..f132451abcc 100644 --- a/playground/backend/internal/db/schema/migration/migration_v001.go +++ b/playground/backend/internal/db/schema/migration_v001.go @@ -13,29 +13,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -package migration +package schema import ( pb "beam.apache.org/playground/backend/internal/api/v1" + ds "beam.apache.org/playground/backend/internal/db/datastore" "beam.apache.org/playground/backend/internal/db/entity" - "beam.apache.org/playground/backend/internal/db/schema" "beam.apache.org/playground/backend/internal/utils" + "cloud.google.com/go/datastore" + "context" ) -type InitialStructure struct { +type migrationV001 struct { + ds.MigrationBase } -func (is *InitialStructure) InitiateData(args *schema.DBArgs) error { - //init schema versions - schemaEntity := &entity.SchemaEntity{Descr: is.GetDescription()} - if err := args.Db.PutSchemaVersion(args.Ctx, is.GetVersion(), schemaEntity); err != nil { - return err +func GetMigration_v001() ds.Migration { + return &migrationV001{ + MigrationBase: ds.MigrationBase{ + Version: 1, + Description: "Data initialization: a schema version, SDKs", + }, } +} - //init sdks +func (m migrationV001) Apply(ctx context.Context, tx *datastore.Transaction, sdkConfigPath string) error { + // Init sdks var sdkEntities []*entity.SDKEntity sdkConfig := new(SdkConfig) - if err := utils.ReadYamlFile(args.AppEnv.SdkConfigPath(), sdkConfig); err != nil { + if err := utils.ReadYamlFile(sdkConfigPath, sdkConfig); err != nil { return err } for _, sdk := range pb.Sdk_name { @@ -48,7 +54,7 @@ func (is *InitialStructure) InitiateData(args *schema.DBArgs) error { DefaultExample: defaultExample, }) } - if err := args.Db.PutSDKs(args.Ctx, sdkEntities); err != nil { + if err := ds.TxPutSDKs(ctx, tx, sdkEntities); err != nil { return err } @@ -82,11 +88,3 @@ func getDefaultExample(config *SdkConfig, sdk string) string { return "" } } - -func (is *InitialStructure) GetVersion() string { - return "0.0.1" -} - -func (is InitialStructure) GetDescription() string { - return "Data initialization: a schema version, SDKs" -} diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/internal/db/schema/migrations.go similarity index 81% copy from playground/backend/internal/db/entity/schema.go copy to playground/backend/internal/db/schema/migrations.go index 43aaacfca34..690a57509a9 100644 --- a/playground/backend/internal/db/entity/schema.go +++ b/playground/backend/internal/db/schema/migrations.go @@ -13,8 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package entity +package schema -type SchemaEntity struct { - Descr string `datastore:"descr,noindex"` +import ds "beam.apache.org/playground/backend/internal/db/datastore" + +// Migrations List of all migrations +var Migrations = []ds.Migration{ + GetMigration_v001(), } diff --git a/playground/backend/internal/db/schema/version.go b/playground/backend/internal/db/schema/version.go deleted file mode 100644 index 457011aa9e9..00000000000 --- a/playground/backend/internal/db/schema/version.go +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package schema - -import ( - "beam.apache.org/playground/backend/internal/db" - "beam.apache.org/playground/backend/internal/environment" - "beam.apache.org/playground/backend/internal/logger" - "context" - "sort" -) - -type DBArgs struct { - Ctx context.Context - Db db.Database - AppEnv *environment.ApplicationEnvs - Props *environment.Properties -} - -type DBSchema struct { - args *DBArgs - versions []Version -} - -func New(ctx context.Context, db db.Database, appEnv *environment.ApplicationEnvs, props *environment.Properties, versions []Version) *DBSchema { - return &DBSchema{ - args: &DBArgs{ctx, db, appEnv, props}, - versions: versions, - } -} - -func (ds *DBSchema) InitiateData() (string, error) { - var versions []string - for _, ver := range ds.versions { - if err := ver.InitiateData(ds.args); err != nil { - logger.Errorf("DBSchema: InitiateData() error during the data initialization, err: %s", err.Error()) - return "", err - } - versions = append(versions, ver.GetVersion()) - } - sort.Strings(versions) - return versions[len(versions)-1], nil -} - -type Version interface { - GetVersion() string - GetDescription() string - InitiateData(args *DBArgs) error -} diff --git a/playground/backend/internal/environment/application.go b/playground/backend/internal/environment/application.go index 1b3e1e37bb0..e61fcea5563 100644 --- a/playground/backend/internal/environment/application.go +++ b/playground/backend/internal/environment/application.go @@ -101,7 +101,7 @@ type ApplicationEnvs struct { pipelinesFolder string // schemaVersion is the database schema version - schemaVersion string + schemaVersion int // sdkConfigPath is a sdk configuration file sdkConfigPath string @@ -117,26 +117,48 @@ type ApplicationEnvs struct { // cacheRequestTimeout is timeout to request data from cache cacheRequestTimeout time.Duration + + // cleanupSnippetsFunctionsUrl is the url to cleanup snippets functions + cleanupSnippetsFunctionsUrl string + + // putSnippetFunctionUrl is the url to put snippet function + putSnippetFunctionsUrl string + + // incrementSnippetViewsFunctionsUrl is the url to increment snippet views + incrementSnippetViewsFunctionsUrl string } // NewApplicationEnvs constructor for ApplicationEnvs func NewApplicationEnvs( - workingDir, launchSite, projectId, pipelinesFolder, sdkConfigPath, propertyPath, kafkaEmulatorExecutablePath, datasetsPath string, + workingDir, + launchSite, + projectId, + pipelinesFolder, + sdkConfigPath, + propertyPath, + kafkaEmulatorExecutablePath, + datasetsPath, + cleanupSnippetsFunctionsUrl, + putSnippetFunctionsUrl, + incrementSnippetViewsFunctionsUrl string, cacheEnvs *CacheEnvs, pipelineExecuteTimeout, cacheRequestTimeout time.Duration, ) *ApplicationEnvs { return &ApplicationEnvs{ - workingDir: workingDir, - cacheEnvs: cacheEnvs, - pipelineExecuteTimeout: pipelineExecuteTimeout, - launchSite: launchSite, - projectId: projectId, - pipelinesFolder: pipelinesFolder, - sdkConfigPath: sdkConfigPath, - propertyPath: propertyPath, - datasetsPath: datasetsPath, - kafkaEmulatorExecutablePath: kafkaEmulatorExecutablePath, - cacheRequestTimeout: cacheRequestTimeout, + workingDir: workingDir, + cacheEnvs: cacheEnvs, + pipelineExecuteTimeout: pipelineExecuteTimeout, + launchSite: launchSite, + projectId: projectId, + pipelinesFolder: pipelinesFolder, + sdkConfigPath: sdkConfigPath, + propertyPath: propertyPath, + datasetsPath: datasetsPath, + kafkaEmulatorExecutablePath: kafkaEmulatorExecutablePath, + cacheRequestTimeout: cacheRequestTimeout, + cleanupSnippetsFunctionsUrl: cleanupSnippetsFunctionsUrl, + putSnippetFunctionsUrl: putSnippetFunctionsUrl, + incrementSnippetViewsFunctionsUrl: incrementSnippetViewsFunctionsUrl, } } @@ -171,7 +193,7 @@ func (ae *ApplicationEnvs) PipelinesFolder() string { } // SchemaVersion returns the database schema version -func (ae *ApplicationEnvs) SchemaVersion() string { +func (ae *ApplicationEnvs) SchemaVersion() int { return ae.schemaVersion } @@ -186,7 +208,7 @@ func (ae *ApplicationEnvs) PropertyPath() string { } // SetSchemaVersion sets the database schema version -func (ae *ApplicationEnvs) SetSchemaVersion(schemaVersion string) { +func (ae *ApplicationEnvs) SetSchemaVersion(schemaVersion int) { ae.schemaVersion = schemaVersion } @@ -203,3 +225,18 @@ func (ae *ApplicationEnvs) DatasetsPath() string { func (ae *ApplicationEnvs) KafkaExecutablePath() string { return ae.kafkaEmulatorExecutablePath } + +// CleanupSnippetsFunctionsUrl returns the url to cleanup snippets functions +func (ae *ApplicationEnvs) CleanupSnippetsFunctionsUrl() string { + return ae.cleanupSnippetsFunctionsUrl +} + +// PutSnippetFunctionUrl returns the url to put snippet functions +func (ae *ApplicationEnvs) PutSnippetFunctionsUrl() string { + return ae.putSnippetFunctionsUrl +} + +// IncrementSnippetViewsFunctionsUrl returns the url to increment snippet views +func (ae *ApplicationEnvs) IncrementSnippetViewsFunctionsUrl() string { + return ae.incrementSnippetViewsFunctionsUrl +} diff --git a/playground/backend/internal/environment/environment_service.go b/playground/backend/internal/environment/environment_service.go index 6536a4656a4..27e4e0d511a 100644 --- a/playground/backend/internal/environment/environment_service.go +++ b/playground/backend/internal/environment/environment_service.go @@ -32,47 +32,53 @@ import ( ) const ( - serverIpKey = "SERVER_IP" - serverPortKey = "SERVER_PORT" - beamSdkKey = "BEAM_SDK" - beamVersionKey = "BEAM_VERSION" - workingDirKey = "APP_WORK_DIR" - preparedModDirKey = "PREPARED_MOD_DIR" - numOfParallelJobsKey = "NUM_PARALLEL_JOBS" - cacheTypeKey = "CACHE_TYPE" - cacheAddressKey = "CACHE_ADDRESS" - beamPathKey = "BEAM_PATH" - cacheKeyExpirationTimeKey = "KEY_EXPIRATION_TIME" - pipelineExecuteTimeoutKey = "PIPELINE_EXPIRATION_TIMEOUT" - protocolTypeKey = "PROTOCOL_TYPE" - launchSiteKey = "LAUNCH_SITE" - projectIdKey = "GOOGLE_CLOUD_PROJECT" - pipelinesFolderKey = "PIPELINES_FOLDER_NAME" - defaultPipelinesFolder = "executable_files" - defaultLaunchSite = "local" - defaultProtocol = "HTTP" - defaultIp = "localhost" - defaultPort = 8080 - defaultSdk = pb.Sdk_SDK_UNSPECIFIED - defaultBeamVersion = "<unknown>" - defaultBeamJarsPath = "/opt/apache/beam/jars/*" - defaultDatasetsPath = "/opt/playground/backend/datasets" - defaultKafkaEmulatorExecutablePath = "/opt/playground/backend/kafka-emulator/beam-playground-kafka-emulator.jar" - defaultCacheType = "local" - defaultCacheAddress = "localhost:6379" - defaultCacheKeyExpirationTime = time.Minute * 15 - defaultPipelineExecuteTimeout = time.Minute * 10 - jsonExt = ".json" - configFolderName = "configs" - defaultNumOfParallelJobs = 20 - SDKConfigPathKey = "SDK_CONFIG" - defaultSDKConfigPath = "../sdks.yaml" - propertyPathKey = "PROPERTY_PATH" - datasetsPathKey = "DATASETS_PATH" - kafkaEmulatorExecutablePathKey = "KAFKA_EMULATOR_EXECUTABLE_PATH" - defaultPropertyPath = "." - cacheRequestTimeoutKey = "CACHE_REQUEST_TIMEOUT" - defaultCacheRequestTimeout = time.Second * 5 + serverIpKey = "SERVER_IP" + serverPortKey = "SERVER_PORT" + beamSdkKey = "BEAM_SDK" + beamVersionKey = "BEAM_VERSION" + workingDirKey = "APP_WORK_DIR" + preparedModDirKey = "PREPARED_MOD_DIR" + numOfParallelJobsKey = "NUM_PARALLEL_JOBS" + cacheTypeKey = "CACHE_TYPE" + cacheAddressKey = "CACHE_ADDRESS" + beamPathKey = "BEAM_PATH" + cacheKeyExpirationTimeKey = "KEY_EXPIRATION_TIME" + pipelineExecuteTimeoutKey = "PIPELINE_EXPIRATION_TIMEOUT" + protocolTypeKey = "PROTOCOL_TYPE" + launchSiteKey = "LAUNCH_SITE" + projectIdKey = "GOOGLE_CLOUD_PROJECT" + pipelinesFolderKey = "PIPELINES_FOLDER_NAME" + defaultPipelinesFolder = "executable_files" + defaultLaunchSite = "local" + defaultProtocol = "HTTP" + defaultIp = "localhost" + defaultPort = 8080 + defaultSdk = pb.Sdk_SDK_UNSPECIFIED + defaultBeamVersion = "<unknown>" + defaultBeamJarsPath = "/opt/apache/beam/jars/*" + defaultDatasetsPath = "/opt/playground/backend/datasets" + defaultKafkaEmulatorExecutablePath = "/opt/playground/backend/kafka-emulator/beam-playground-kafka-emulator.jar" + defaultCacheType = "local" + defaultCacheAddress = "localhost:6379" + defaultCacheKeyExpirationTime = time.Minute * 15 + defaultPipelineExecuteTimeout = time.Minute * 10 + jsonExt = ".json" + configFolderName = "configs" + defaultNumOfParallelJobs = 20 + SDKConfigPathKey = "SDK_CONFIG" + defaultSDKConfigPath = "../sdks.yaml" + propertyPathKey = "PROPERTY_PATH" + datasetsPathKey = "DATASETS_PATH" + kafkaEmulatorExecutablePathKey = "KAFKA_EMULATOR_EXECUTABLE_PATH" + defaultPropertyPath = "." + cacheRequestTimeoutKey = "CACHE_REQUEST_TIMEOUT" + defaultCacheRequestTimeout = time.Second * 5 + cleanupSnippetsFunctionsUrlKey = "CLEANUP_SNIPPETS_FUNCTIONS_URL" + defaultCleanupSnippetsFunctionsUrl = "http://cleanup_snippets:8080/" + putSnippetFunctionsUrlKey = "PUT_SNIPPET_FUNCTIONS_URL" + defaultPutSnippetFunctionsUrl = "http://put_snippet:8080/" + incrementSnippetViewsFunctionsUrlKey = "INCREMENT_SNIPPET_VIEWS_FUNCTIONS_URL" + defaultIncrementSnippetViewsFunctionsUrl = "http://increment_snippet_views:8080/" ) // Environment operates with environment structures: NetworkEnvs, BeamEnvs, ApplicationEnvs @@ -120,9 +126,31 @@ func GetApplicationEnvsFromOsEnvs() (*ApplicationEnvs, error) { datasetsPath := getEnv(datasetsPathKey, defaultDatasetsPath) kafkaEmulatorExecutablePath := getEnv(kafkaEmulatorExecutablePathKey, defaultKafkaEmulatorExecutablePath) cacheRequestTimeout := getEnvAsDuration(cacheRequestTimeoutKey, defaultCacheRequestTimeout, "couldn't convert provided cache request timeout. Using default %s\n") + cleanupSnippetsFunctionsUrl := getEnv(cleanupSnippetsFunctionsUrlKey, defaultCleanupSnippetsFunctionsUrl) + putSnippetFunctionsUrl := getEnv(putSnippetFunctionsUrlKey, defaultPutSnippetFunctionsUrl) + incrementSnippetViewsFunctionsUrl := getEnv(incrementSnippetViewsFunctionsUrlKey, defaultIncrementSnippetViewsFunctionsUrl) if value, present := os.LookupEnv(workingDirKey); present { - return NewApplicationEnvs(value, launchSite, projectId, pipelinesFolder, sdkConfigPath, propertyPath, kafkaEmulatorExecutablePath, datasetsPath, NewCacheEnvs(cacheType, cacheAddress, cacheExpirationTime), pipelineExecuteTimeout, cacheRequestTimeout), nil + return NewApplicationEnvs( + value, + launchSite, + projectId, + pipelinesFolder, + sdkConfigPath, + propertyPath, + kafkaEmulatorExecutablePath, + datasetsPath, + cleanupSnippetsFunctionsUrl, + putSnippetFunctionsUrl, + incrementSnippetViewsFunctionsUrl, + NewCacheEnvs( + cacheType, + cacheAddress, + cacheExpirationTime, + ), + pipelineExecuteTimeout, + cacheRequestTimeout, + ), nil } return nil, errors.New("APP_WORK_DIR env should be provided with os.env") } diff --git a/playground/backend/internal/environment/environment_service_test.go b/playground/backend/internal/environment/environment_service_test.go index ae8cf4a06e4..f74578aed57 100644 --- a/playground/backend/internal/environment/environment_service_test.go +++ b/playground/backend/internal/environment/environment_service_test.go @@ -103,9 +103,28 @@ func TestNewEnvironment(t *testing.T) { want *Environment }{ {name: "Create env service with default envs", want: &Environment{ - NetworkEnvs: *NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol), - BeamSdkEnvs: *NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0), - ApplicationEnvs: *NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout), + NetworkEnvs: *NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol), + BeamSdkEnvs: *NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0), + ApplicationEnvs: *NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout, + ), }}, } for _, tt := range tests { @@ -113,7 +132,26 @@ func TestNewEnvironment(t *testing.T) { if got := NewEnvironment( *NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol), *NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0), - *NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout)); !reflect.DeepEqual(got, tt.want) { + *NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout, + )); !reflect.DeepEqual(got, tt.want) { t.Errorf("NewEnvironment() = %v, want %v", got, tt.want) } }) @@ -223,8 +261,27 @@ func Test_getApplicationEnvsFromOsEnvs(t *testing.T) { envsToSet map[string]string }{ { - name: "Working dir is provided", - want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout), + name: "Working dir is provided", + want: NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout, + ), wantErr: false, envsToSet: map[string]string{workingDirKey: "/app", launchSiteKey: defaultLaunchSite, projectIdKey: defaultProjectId}, }, @@ -234,26 +291,101 @@ func Test_getApplicationEnvsFromOsEnvs(t *testing.T) { wantErr: true, }, { - name: "CacheKeyExpirationTimeKey is set with the correct value", - want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, convertedTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout), + name: "CacheKeyExpirationTimeKey is set with the correct value", + want: NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + convertedTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout), wantErr: false, envsToSet: map[string]string{workingDirKey: "/app", cacheKeyExpirationTimeKey: hour}, }, { - name: "CacheKeyExpirationTimeKey is set with the incorrect value", - want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout), + name: "CacheKeyExpirationTimeKey is set with the incorrect value", + want: NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout, + ), wantErr: false, envsToSet: map[string]string{workingDirKey: "/app", cacheKeyExpirationTimeKey: "1"}, }, { - name: "CacheKeyExpirationTimeKey is set with the correct value", - want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, convertedTime, defaultCacheRequestTimeout), + name: "CacheKeyExpirationTimeKey is set with the correct value", + want: NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + convertedTime, + defaultCacheRequestTimeout, + ), wantErr: false, envsToSet: map[string]string{workingDirKey: "/app", pipelineExecuteTimeoutKey: hour}, }, { - name: "PipelineExecuteTimeoutKey is set with the incorrect value", - want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout), + name: "PipelineExecuteTimeoutKey is set with the incorrect value", + want: NewApplicationEnvs( + "/app", + defaultLaunchSite, + defaultProjectId, + defaultPipelinesFolder, + defaultSDKConfigPath, + defaultPropertyPath, + defaultKafkaEmulatorExecutablePath, + defaultDatasetsPath, + defaultCleanupSnippetsFunctionsUrl, + defaultPutSnippetFunctionsUrl, + defaultIncrementSnippetViewsFunctionsUrl, + &CacheEnvs{ + defaultCacheType, + defaultCacheAddress, + defaultCacheKeyExpirationTime, + }, + defaultPipelineExecuteTimeout, + defaultCacheRequestTimeout, + ), wantErr: false, envsToSet: map[string]string{workingDirKey: "/app", pipelineExecuteTimeoutKey: "1"}, }, diff --git a/playground/backend/internal/environment/property.go b/playground/backend/internal/environment/property.go index 8e9d034c097..db5a2917f9b 100644 --- a/playground/backend/internal/environment/property.go +++ b/playground/backend/internal/environment/property.go @@ -34,8 +34,6 @@ type Properties struct { IdLength int8 `mapstructure:"id_length"` // RemovingUnusedSnptsCron is the cron expression for the scheduled task to remove unused snippets RemovingUnusedSnptsCron string `mapstructure:"removing_unused_snippets_cron"` - // RemovingUnusedSnptsDays is the number of days after which a snippet becomes unused - RemovingUnusedSnptsDays int32 `mapstructure:"removing_unused_snippets_days"` } func NewProperties(configPath string) (*Properties, error) { diff --git a/playground/backend/internal/environment/property_test.go b/playground/backend/internal/environment/property_test.go index 26d6d7c26e2..794eb1ec405 100644 --- a/playground/backend/internal/environment/property_test.go +++ b/playground/backend/internal/environment/property_test.go @@ -47,8 +47,7 @@ func TestNew(t *testing.T) { if props.Salt != "Beam playground salt" || props.MaxSnippetSize != 1000000 || props.IdLength != 11 || - props.RemovingUnusedSnptsCron != "0 0 0 1 */1 *" || - props.RemovingUnusedSnptsDays != 180 { + props.RemovingUnusedSnptsCron != "0 0 0 1 */1 *" { t.Errorf("NewProperties(): unexpected result") } }) diff --git a/playground/backend/internal/external_functions/external_functions_component.go b/playground/backend/internal/external_functions/external_functions_component.go new file mode 100644 index 00000000000..5e6e416e67c --- /dev/null +++ b/playground/backend/internal/external_functions/external_functions_component.go @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external_functions + +import ( + "beam.apache.org/playground/backend/internal/utils" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "beam.apache.org/playground/backend/internal/db/entity" + "beam.apache.org/playground/backend/internal/environment" + "beam.apache.org/playground/backend/internal/logger" +) + +type ExternalFunctions interface { + // CleanupSnippets removes old snippets from the database. + CleanupSnippets(ctx context.Context) error + + // PutSnippet puts the snippet to the database. + PutSnippet(ctx context.Context, snipId string, snippet *entity.Snippet) error + + // IncrementSnippetViews increments the number of views for the snippet. + IncrementSnippetViews(ctx context.Context, snipId string) error +} + +type externalFunctionsComponent struct { + cleanupSnippetsFunctionsUrl string + putSnippetFunctionsUrl string + incrementSnippetViewsFunctionsUrl string +} + +func NewExternalFunctionsComponent(appEnvs environment.ApplicationEnvs) ExternalFunctions { + return &externalFunctionsComponent{ + cleanupSnippetsFunctionsUrl: appEnvs.CleanupSnippetsFunctionsUrl(), + putSnippetFunctionsUrl: appEnvs.PutSnippetFunctionsUrl(), + incrementSnippetViewsFunctionsUrl: appEnvs.IncrementSnippetViewsFunctionsUrl(), + } +} + +func makePostRequest(ctx context.Context, requestUrl string, body any) error { + var bodyReader io.Reader = nil + + if body != nil { + bodyJson, err := json.Marshal(body) + if err != nil { + logger.Errorf("makePostRequest(): Couldn't marshal the body, err: %s\n", err.Error()) + return err + } + + bodyReader = bytes.NewReader(bodyJson) + } else { + bodyReader = bytes.NewReader([]byte("{}")) + } + + request, err := http.NewRequestWithContext(ctx, "POST", requestUrl, bodyReader) + if err != nil { + logger.Errorf("makePostRequest(): Couldn't create the request, err: %s\n", err.Error()) + return err + } + + request.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(request) + if err != nil { + logger.Errorf("makePostRequest(): Couldn't make POST request to the %s, err: %s\n", requestUrl, err.Error()) + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status code: %d", resp.StatusCode) + } + + return nil +} + +func (c *externalFunctionsComponent) CleanupSnippets(ctx context.Context) error { + namespace := utils.GetNamespace(ctx) + requestUrl := fmt.Sprintf("%s?namespace=%s", c.cleanupSnippetsFunctionsUrl, namespace) + + if err := makePostRequest(ctx, requestUrl, nil); err != nil { + logger.Errorf("CleanupSnippets(): Couldn't cleanup snippets, err: %s\n", err.Error()) + return err + } + + return nil +} + +func (c *externalFunctionsComponent) PutSnippet(ctx context.Context, snipId string, snippet *entity.Snippet) error { + namespace := utils.GetNamespace(ctx) + requestUrl := fmt.Sprintf("%s?namespace=%s&snipId=%s", c.putSnippetFunctionsUrl, namespace, snipId) + + if err := makePostRequest(ctx, requestUrl, snippet); err != nil { + logger.Errorf("DeleteObsoleteSnippets(): Couldn't delete obsolete snippets, err: %s\n", err.Error()) + return err + } + + return nil +} + +func (c *externalFunctionsComponent) IncrementSnippetViews(ctx context.Context, snipId string) error { + namespace := utils.GetNamespace(ctx) + requestUrl := fmt.Sprintf("%s?namespace=%s&snipId=%s", c.incrementSnippetViewsFunctionsUrl, namespace, snipId) + + if err := makePostRequest(ctx, requestUrl, nil); err != nil { + logger.Errorf("IncrementSnippetViews(): Couldn't increment snippet views, err: %s\n", err.Error()) + return err + } + + return nil +} diff --git a/playground/backend/internal/tasks/task.go b/playground/backend/internal/tasks/task.go index c77b4a06133..3d214bbdab3 100644 --- a/playground/backend/internal/tasks/task.go +++ b/playground/backend/internal/tasks/task.go @@ -16,12 +16,12 @@ package tasks import ( + "beam.apache.org/playground/backend/internal/external_functions" "context" "time" "github.com/procyon-projects/chrono" - "beam.apache.org/playground/backend/internal/db" "beam.apache.org/playground/backend/internal/logger" ) @@ -34,11 +34,11 @@ func New(ctx context.Context) *ScheduledTask { return &ScheduledTask{ctx: ctx, taskScheduler: chrono.NewDefaultTaskScheduler()} } -func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, dayDiff int32, db db.Database) error { +func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, externalFunction external_functions.ExternalFunctions) error { task, err := st.taskScheduler.ScheduleWithCron(func(ctx context.Context) { logger.Info("ScheduledTask: StartRemovingExtraSnippets() is running...\n") startDate := time.Now() - if err := db.DeleteUnusedSnippets(ctx, dayDiff); err != nil { + if err := externalFunction.CleanupSnippets(ctx); err != nil { logger.Errorf("ScheduledTask: StartRemovingExtraSnippets() error during deleting unused snippets, err: %s\n", err.Error()) } diffTime := time.Now().Sub(startDate).Milliseconds() diff --git a/playground/backend/internal/utils/datastore_utils.go b/playground/backend/internal/utils/datastore_utils.go index 757350eb8fd..a4706189506 100644 --- a/playground/backend/internal/utils/datastore_utils.go +++ b/playground/backend/internal/utils/datastore_utils.go @@ -16,6 +16,7 @@ package utils import ( + "beam.apache.org/playground/backend/internal/logger" "context" "os" "strconv" @@ -93,7 +94,11 @@ func getNameKey(ctx context.Context, kind, id string, parentId *datastore.Key) * } func GetNamespace(ctx context.Context) string { - namespace, ok := ctx.Value(constants.DatastoreNamespaceKey).(string) + namespaceValue := ctx.Value(constants.DatastoreNamespaceKey) + namespace, ok := namespaceValue.(string) + if namespaceValue != nil && !ok { + logger.Warnf("GetNamespace(): %s value is set in context, but is not a string", constants.DatastoreNamespaceKey) + } if !ok { namespace, ok = os.LookupEnv(constants.DatastoreNamespaceKey) if !ok { diff --git a/playground/backend/playground_functions/Dockerfile b/playground/backend/playground_functions/Dockerfile new file mode 100644 index 00000000000..a4045c02131 --- /dev/null +++ b/playground/backend/playground_functions/Dockerfile @@ -0,0 +1,38 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +# This Dockerfile is only for local testing + +FROM golang:1.20-alpine as build + +COPY . /app +WORKDIR /app/playground_functions + +RUN ls -la + +RUN go mod download + +RUN go build -o /app/cloudfunction ./cmd + +FROM alpine:3.17 + +COPY --from=build /app/cloudfunction /app/cloudfunction + +EXPOSE 8080 + +ENTRYPOINT ["/app/cloudfunction"] diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/cmd/main.go similarity index 66% copy from playground/backend/internal/db/entity/schema.go copy to playground/backend/playground_functions/cmd/main.go index 43aaacfca34..f4946694bfc 100644 --- a/playground/backend/internal/db/entity/schema.go +++ b/playground/backend/playground_functions/cmd/main.go @@ -13,8 +13,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package entity +package main -type SchemaEntity struct { - Descr string `datastore:"descr,noindex"` +import ( + "log" + "os" + + _ "beam.apache.org/playground/backend" + "github.com/GoogleCloudPlatform/functions-framework-go/funcframework" +) + +func main() { + // Use PORT environment variable, or default to 8080. + port := "8080" + if envPort := os.Getenv("PORT"); envPort != "" { + port = envPort + } + if err := funcframework.Start(port); err != nil { + log.Fatalf("funcframework.Start: %v\n", err) + } } diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/func_enviornment.go similarity index 69% copy from playground/backend/internal/db/entity/schema.go copy to playground/backend/playground_functions/func_enviornment.go index 43aaacfca34..f86b55bfe84 100644 --- a/playground/backend/internal/db/entity/schema.go +++ b/playground/backend/playground_functions/func_enviornment.go @@ -13,8 +13,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -package entity +package playground_functions -type SchemaEntity struct { - Descr string `datastore:"descr,noindex"` +import "os" + +type Environment interface { + GetProjectId() string +} + +type environment struct { + projectID string +} + +func GetEnvironment() Environment { + projectId := os.Getenv("GOOGLE_CLOUD_PROJECT") + return &environment{ + projectID: projectId, + } +} + +func (e *environment) GetProjectId() string { + return e.projectID } diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/middleware.go similarity index 63% copy from playground/backend/internal/db/entity/schema.go copy to playground/backend/playground_functions/middleware.go index 43aaacfca34..8927ea1ec82 100644 --- a/playground/backend/internal/db/entity/schema.go +++ b/playground/backend/playground_functions/middleware.go @@ -13,8 +13,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package entity +package playground_functions -type SchemaEntity struct { - Descr string `datastore:"descr,noindex"` +import "net/http" + +// EnsureMethod is a middleware method which will only allow requests with the specified method to pass through. +func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method == method { + next(w, r) + } else { + w.WriteHeader(http.StatusMethodNotAllowed) + } + } + } } diff --git a/playground/backend/properties.yaml b/playground/backend/properties.yaml index 12f5adb2218..67d969ce370 100644 --- a/playground/backend/properties.yaml +++ b/playground/backend/properties.yaml @@ -23,5 +23,3 @@ id_length: 11 # Check your cron expression here: https://crontab.cronhub.io/ # Time trigger for the scheduled task will be updated every time the application is restarted removing_unused_snippets_cron: 0 0 0 1 */1 * -# Number of days after which a snippet becomes unused -removing_unused_snippets_days: 180 diff --git a/playground/docker-compose.local.yaml b/playground/docker-compose.local.yaml index 92121a2f0c5..a6c38925e31 100644 --- a/playground/docker-compose.local.yaml +++ b/playground/docker-compose.local.yaml @@ -29,6 +29,45 @@ services: ports: - "8081:8081" + cleanup_snippets: + build: + context: ./backend + dockerfile: ./playground_functions/Dockerfile + environment: + GOOGLE_CLOUD_PROJECT: test + DATASTORE_EMULATOR_HOST: datastore:8081 + FUNCTION_TARGET: cleanupSnippets + expose: + - "8080" + depends_on: + - datastore + + put_snippet: + build: + context: ./backend + dockerfile: ./playground_functions/Dockerfile + environment: + GOOGLE_CLOUD_PROJECT: test + DATASTORE_EMULATOR_HOST: datastore:8081 + FUNCTION_TARGET: putSnippet + expose: + - "8080" + depends_on: + - datastore + + increment_snippet_views: + build: + context: ./backend + dockerfile: ./playground_functions/Dockerfile + environment: + GOOGLE_CLOUD_PROJECT: test + DATASTORE_EMULATOR_HOST: datastore:8081 + FUNCTION_TARGET: incrementSnippetViews + expose: + - "8080" + depends_on: + - datastore + router: image: apache/beam_playground-backend-router environment: @@ -38,11 +77,15 @@ services: CACHE_ADDRESS: redis:6379 SDK_CONFIG: /opt/playground/backend/sdks-emulator.yaml SERVER_PORT: 8082 + APPLY_MIGRATIONS: "True" ports: - "8082:8082" depends_on: - redis - datastore + - increment_snippet_views + - put_snippet + - cleanup_snippets go_runner: image: apache/beam_playground-backend-go diff --git a/playground/index.yaml b/playground/index.yaml index 184b8ad3a2d..85461605c18 100644 --- a/playground/index.yaml +++ b/playground/index.yaml @@ -23,4 +23,3 @@ indexes: properties: - name: persistenceKey - name: numberOfFiles - diff --git a/playground/infrastructure/helm-playground/templates/deployment-router.yml b/playground/infrastructure/helm-playground/templates/deployment-router.yml index 7b2d20dca2c..e505a849589 100644 --- a/playground/infrastructure/helm-playground/templates/deployment-router.yml +++ b/playground/infrastructure/helm-playground/templates/deployment-router.yml @@ -44,6 +44,12 @@ spec: value: "5" - name: DATASTORE_NAMESPACE value: {{ .Values.datastore_name }} + - name: CLEANUP_SNIPPETS_FUNCTIONS_URL + value: {{ .Values.func_clean }} + - name: PUT_SNIPPET_FUNCTIONS_URL + value: {{ .Values.func_put }} + - name: INCREMENT_SNIPPET_VIEWS_FUNCTIONS_URL + value: {{ .Values.func_view }} livenessProbe: httpGet: path: /liveness diff --git a/playground/terraform/README.md b/playground/terraform/README.md index 4a3db70600c..3ff1b62bef3 100644 --- a/playground/terraform/README.md +++ b/playground/terraform/README.md @@ -31,7 +31,9 @@ Ensure that the account has at least following privileges: - App Engine Creator - Artifact Registry Administrator - Cloud Datastore Index Admin + - Cloud Datastore User - Cloud Memorystore Redis Admin + - Cloud Functions Developer - Compute Admin - Create Service Accounts - DNS Administrator @@ -55,6 +57,7 @@ Ensure that the account has at least following privileges: * [Terraform](https://www.terraform.io/downloads) * [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) * [Kubectl authentication](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) +* [GO](https://go.dev/doc/install) 6. Apache Beam Git repository cloned locally @@ -76,10 +79,11 @@ redis_tier = "BASIC" # Redis tier type. Options: min_count = 2 # Min node count for the GKE cluster max_count = 6 # Max node count for the GKE cluster skip_appengine_deploy = false # AppEngine flag - defined if AppEngine and Datastore need to be installed. Should be "true" if AppEngine and Datastore were installed before -ip-address-name = "playground-static-ip" # GCP Static IP Address name +ip_address_name = "playground-static-ip" # GCP Static IP Address name repository_id = "playground-artifacts" # GCP Artifact repository name for Playground images -service_account_id = "playground-gke-account" # GCP Service account name +service_account_id = "playground-gke-sa" # GCP Service account name gke_machine_type = "e2-standard-8" # Machine type for GKE Nodes +env = "prod" # Environment. The same value as for <environment_name> parameter ``` * `state.tfbackend` environment variables: diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 8e1e5b48fca..4fdb1007b35 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -267,6 +267,51 @@ tasks.register<TerraformTask>("setPlaygroundStaticIpAddressName") { } } +tasks.register<TerraformTask>("setPlaygroundFunctionCleanupUrl") { + group = "deploy" + + dependsOn("terraformInit") + dependsOn("terraformRef") + + args("output", "playground_function_cleanup_url") + standardOutput = ByteArrayOutputStream() + + doLast { + project.rootProject.extra["playground_function_cleanup_url"] = + standardOutput.toString().trim().replace("\"", "") + } +} + +tasks.register<TerraformTask>("setPlaygroundFunctionPutUrl") { + group = "deploy" + + dependsOn("terraformInit") + dependsOn("terraformRef") + + args("output", "playground_function_put_url") + standardOutput = ByteArrayOutputStream() + + doLast { + project.rootProject.extra["playground_function_put_url"] = + standardOutput.toString().trim().replace("\"", "") + } +} + +tasks.register<TerraformTask>("setPlaygroundFunctionViewUrl") { + group = "deploy" + + dependsOn("terraformInit") + dependsOn("terraformRef") + + args("output", "playground_function_view_url") + standardOutput = ByteArrayOutputStream() + + doLast { + project.rootProject.extra["playground_function_view_url"] = + standardOutput.toString().trim().replace("\"", "") + } +} + tasks.register("takeConfig") { group = "deploy" @@ -274,6 +319,9 @@ tasks.register("takeConfig") { dependsOn("setPlaygroundRedisIp") dependsOn("setPlaygroundGkeProject") dependsOn("setPlaygroundStaticIpAddressName") + dependsOn("setPlaygroundFunctionCleanupUrl") + dependsOn("setPlaygroundFunctionPutUrl") + dependsOn("setPlaygroundFunctionViewUrl") doLast { var d_tag = "" @@ -314,6 +362,9 @@ tasks.register("takeConfig") { val registry = project.rootProject.extra["docker-repository-root"] val ipaddrname = project.rootProject.extra["playground_static_ip_address_name"] val datastore_name = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else "" + val pgfuncclean = project.rootProject.extra["playground_function_cleanup_url"] + val pgfuncput = project.rootProject.extra["playground_function_put_url"] + val pgfuncview = project.rootProject.extra["playground_function_view_url"] file.appendText( """ @@ -325,11 +376,32 @@ static_ip_name: ${ipaddrname} tag: $d_tag datastore_name: ${datastore_name} dns_name: ${dns_name} +func_clean: ${pgfuncclean} +func_put: ${pgfuncput} +func_view: ${pgfuncview} """ ) } } + +task("applyMigrations") { + doLast { + val namespace = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else "" + val projectId = project.rootProject.extra["playground_gke_project"] + val modulePath = project(":playground").projectDir.absolutePath + val sdkConfig = "$modulePath/sdks.yaml" + exec { + workingDir("$modulePath/backend") + executable("go") + args("run", "cmd/migration_tool/migration_tool.go", + "-project-id", projectId, + "-sdk-config", sdkConfig, + "-namespace", namespace) + } + } +} + tasks.register("helmRelease") { group = "deploy" val modulePath = project(":playground").projectDir.absolutePath @@ -339,7 +411,7 @@ tasks.register("helmRelease") { executable("helm") args("upgrade", "--install", "playground", "$helmdir") } - } + } } tasks.register("gkebackend") { @@ -350,15 +422,18 @@ tasks.register("gkebackend") { val pushFrontTask = tasks.getByName("pushFront") val indexcreateTask = tasks.getByName("indexcreate") val helmTask = tasks.getByName("helmRelease") + var applyMigrations = tasks.getByName("applyMigrations") dependsOn(initTask) dependsOn(takeConfigTask) dependsOn(pushBackTask) dependsOn(pushFrontTask) dependsOn(indexcreateTask) + dependsOn(applyMigrations) dependsOn(helmTask) takeConfigTask.mustRunAfter(initTask) pushBackTask.mustRunAfter(takeConfigTask) pushFrontTask.mustRunAfter(pushBackTask) indexcreateTask.mustRunAfter(pushFrontTask) - helmTask.mustRunAfter(indexcreateTask) + applyMigrations.mustRunAfter(indexcreateTask) + helmTask.mustRunAfter(applyMigrations) } diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/api_enable/variables.tf index f0c0e5823f3..41eef1e165f 100644 --- a/playground/terraform/infrastructure/api_enable/variables.tf +++ b/playground/terraform/infrastructure/api_enable/variables.tf @@ -23,5 +23,5 @@ variable "project_id" { variable "services" { description = "Enable necessary APIs in GCP" - default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"] + default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","cloudfunctions.googleapis.com","cloudbuild.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"] } diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/archive_file/main.tf similarity index 80% copy from playground/terraform/infrastructure/ip_address/main.tf copy to playground/terraform/infrastructure/archive_file/main.tf index 3e2a8f3524a..90ce40ea149 100644 --- a/playground/terraform/infrastructure/ip_address/main.tf +++ b/playground/terraform/infrastructure/archive_file/main.tf @@ -17,6 +17,12 @@ # under the License. # -resource "google_compute_global_address" "pg-ip" { - name = var.ip-address-name +data "archive_file" "backend_folder" { + type = "zip" + source_dir = "${path.root}/../backend/" + output_path = "${path.root}/../cloudfunction.zip" + + excludes = [ + "containers" + ] } diff --git a/playground/terraform/infrastructure/cloudfunctions/main.tf b/playground/terraform/infrastructure/cloudfunctions/main.tf new file mode 100644 index 00000000000..690b032f3b8 --- /dev/null +++ b/playground/terraform/infrastructure/cloudfunctions/main.tf @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +locals { + functions = [ + { + name = "playground-function-cleanup-${var.env}" + description = "Playground function cleanup-${var.env}" + entry_point = "cleanupSnippets" + }, + { + name = "playground-function-delete-${var.env}" + description = "Playground function delete-${var.env}" + entry_point = "putSnippet" + }, + { + name = "playground-function-view-${var.env}" + description = "Playground function view-${var.env}" + entry_point = "incrementSnippetViews" + }, + ] +} + +resource "google_cloudfunctions_function" "playground_functions" { + count = length(local.functions) + name = local.functions[count.index].name + description = local.functions[count.index].description + entry_point = local.functions[count.index].entry_point + ingress_settings = "ALLOW_INTERNAL_ONLY" + runtime = "go120" + source_archive_bucket = var.gkebucket + source_archive_object = "cloudfunction.zip" + trigger_http = true + timeout = "540" + available_memory_mb = 2048 + service_account_email = var.service_account_email_cf + environment_variables = { + GOOGLE_CLOUD_PROJECT = var.project_id + } +} + +resource "google_cloudfunctions_function_iam_member" "invokers" { + count = length(local.functions) + project = var.project_id + region = var.region + cloud_function = google_cloudfunctions_function.playground_functions[count.index].name + role = "roles/cloudfunctions.invoker" + member = "allUsers" +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/cloudfunctions/output.tf similarity index 68% copy from playground/terraform/infrastructure/api_enable/variables.tf copy to playground/terraform/infrastructure/cloudfunctions/output.tf index f0c0e5823f3..a55636624d0 100644 --- a/playground/terraform/infrastructure/api_enable/variables.tf +++ b/playground/terraform/infrastructure/cloudfunctions/output.tf @@ -17,11 +17,14 @@ # under the License. # -variable "project_id" { - description = "project_id" +output "playground_function_cleanup_url" { + value = google_cloudfunctions_function.playground_functions[0].https_trigger_url } -variable "services" { - description = "Enable necessary APIs in GCP" - default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"] +output "playground_function_put_url" { + value = google_cloudfunctions_function.playground_functions[1].https_trigger_url +} + +output "playground_function_view_url" { + value = google_cloudfunctions_function.playground_functions[2].https_trigger_url } diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/cloudfunctions/variables.tf similarity index 61% copy from playground/terraform/infrastructure/api_enable/variables.tf copy to playground/terraform/infrastructure/cloudfunctions/variables.tf index f0c0e5823f3..9d3f86f63a6 100644 --- a/playground/terraform/infrastructure/api_enable/variables.tf +++ b/playground/terraform/infrastructure/cloudfunctions/variables.tf @@ -17,11 +17,27 @@ # under the License. # +variable "env" { + description = "CloudFunction Environment" +} + +variable "function_description" { + type = string + default = "Playground function" +} + +variable "gkebucket" { + description = "Bucket name for CloudFunction" +} + +variable "service_account_email_cf" { + description = "Service account email for CloudFunctions" +} + variable "project_id" { - description = "project_id" + description = "The GCP Project ID where Playground Applications will be created" } -variable "services" { - description = "Enable necessary APIs in GCP" - default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"] +variable "region" { + description = "The GCP Project region where Cloudfunctions will be created" } diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/gke_bucket/main.tf similarity index 66% copy from playground/terraform/infrastructure/api_enable/variables.tf copy to playground/terraform/infrastructure/gke_bucket/main.tf index f0c0e5823f3..859a01d4443 100644 --- a/playground/terraform/infrastructure/api_enable/variables.tf +++ b/playground/terraform/infrastructure/gke_bucket/main.tf @@ -17,11 +17,19 @@ # under the License. # -variable "project_id" { - description = "project_id" +resource "google_storage_bucket" "bucket" { + name = "${var.bucket_name}-cloudbuild" + location = var.region + force_destroy = true } -variable "services" { - description = "Enable necessary APIs in GCP" - default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"] +resource "google_storage_bucket_object" "cloudfunction_object" { + name = "cloudfunction.zip" + bucket = google_storage_bucket.bucket.name + + source = "${path.root}/../cloudfunction.zip" + + content_type = "application/zip" + content_encoding = "zip" + } diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/gke_bucket/output.tf similarity index 89% copy from playground/terraform/infrastructure/ip_address/main.tf copy to playground/terraform/infrastructure/gke_bucket/output.tf index 3e2a8f3524a..45da0edd96b 100644 --- a/playground/terraform/infrastructure/ip_address/main.tf +++ b/playground/terraform/infrastructure/gke_bucket/output.tf @@ -17,6 +17,6 @@ # under the License. # -resource "google_compute_global_address" "pg-ip" { - name = var.ip-address-name +output "playground_google_storage_bucket" { + value = google_storage_bucket.bucket.name } diff --git a/playground/terraform/infrastructure/ip_address/variables.tf b/playground/terraform/infrastructure/gke_bucket/variables.tf similarity index 83% copy from playground/terraform/infrastructure/ip_address/variables.tf copy to playground/terraform/infrastructure/gke_bucket/variables.tf index 42f60ed7ae3..45de6ab147c 100644 --- a/playground/terraform/infrastructure/ip_address/variables.tf +++ b/playground/terraform/infrastructure/gke_bucket/variables.tf @@ -17,7 +17,10 @@ # under the License. # -variable "ip-address-name" { - description = "Static IP address name" - default = "pg-static-ip" +variable "region" { + description = "Region of Playground Examples Bucket" +} + +variable "bucket_name" { + description = "Bucket name for CloudFunction" } diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/ip_address/main.tf index 3e2a8f3524a..c649d35a8fe 100644 --- a/playground/terraform/infrastructure/ip_address/main.tf +++ b/playground/terraform/infrastructure/ip_address/main.tf @@ -18,5 +18,5 @@ # resource "google_compute_global_address" "pg-ip" { - name = var.ip-address-name + name = var.ip_address_name } diff --git a/playground/terraform/infrastructure/ip_address/variables.tf b/playground/terraform/infrastructure/ip_address/variables.tf index 42f60ed7ae3..dea28526da6 100644 --- a/playground/terraform/infrastructure/ip_address/variables.tf +++ b/playground/terraform/infrastructure/ip_address/variables.tf @@ -17,7 +17,7 @@ # under the License. # -variable "ip-address-name" { +variable "ip_address_name" { description = "Static IP address name" default = "pg-static-ip" } diff --git a/playground/terraform/infrastructure/main.tf b/playground/terraform/infrastructure/main.tf index edb32a36857..75dfab7799d 100644 --- a/playground/terraform/infrastructure/main.tf +++ b/playground/terraform/infrastructure/main.tf @@ -49,6 +49,29 @@ module "artifact_registry" { location = var.repository_location } +module "gke_bucket" { + depends_on = [module.setup, module.network, module.api_enable, module.ip_address, module.archive_file] + source = "./gke_bucket" + region = var.region + bucket_name = var.state_bucket + +} + +module "archive_file" { + depends_on = [module.setup, module.network, module.api_enable, module.ip_address] + source = "./archive_file" +} + +module "cloudfunctions" { + depends_on = [module.setup, module.network, module.api_enable, module.ip_address, module.gke_bucket] + source = "./cloudfunctions" + gkebucket = module.gke_bucket.playground_google_storage_bucket + project_id = var.project_id + service_account_email_cf = module.setup.service_account_email_cf + region = var.region + env = var.env +} + module "memorystore" { depends_on = [module.setup, module.network, module.api_enable, module.ip_address] source = "./memorystore" @@ -81,7 +104,7 @@ module "gke" { module "ip_address" { source = "./ip_address" depends_on = [module.setup, module.api_enable] - ip-address-name = var.ip-address-name + ip_address_name = var.ip_address_name } module "appengine" { @@ -106,6 +129,7 @@ module "private_dns" { private_zones = [ "gcr.io", "pkg.dev", - "cloud.google.com" + "cloud.google.com", + "cloudfunctions.net" ] -} \ No newline at end of file +} diff --git a/playground/terraform/infrastructure/output.tf b/playground/terraform/infrastructure/output.tf index 38d61b7887f..b4e1e6161dc 100644 --- a/playground/terraform/infrastructure/output.tf +++ b/playground/terraform/infrastructure/output.tf @@ -63,4 +63,16 @@ output "playground_gke_project" { output "playground_static_ip_address_name" { value = module.ip_address.playground_static_ip_address_name -} \ No newline at end of file +} + +output "playground_function_cleanup_url" { + value = module.cloudfunctions.playground_function_cleanup_url +} + +output "playground_function_put_url" { + value = module.cloudfunctions.playground_function_put_url +} + +output "playground_function_view_url" { + value = module.cloudfunctions.playground_function_view_url +} diff --git a/playground/terraform/infrastructure/setup/iam.tf b/playground/terraform/infrastructure/setup/iam.tf index ec31f26bf1b..55bfdf215d6 100644 --- a/playground/terraform/infrastructure/setup/iam.tf +++ b/playground/terraform/infrastructure/setup/iam.tf @@ -52,11 +52,25 @@ resource "google_service_account" "playground_service_account" { display_name = var.service_account_id } +resource "google_service_account" "playground_service_account_cf" { + account_id = "${google_service_account.playground_service_account.account_id}-cf" + display_name = "${google_service_account.playground_service_account.account_id}-cf" +} + resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/container.admin", "roles/artifactregistry.reader", "roles/datastore.owner", "roles/redis.admin", + "roles/container.developer", "roles/artifactregistry.reader", "roles/datastore.viewer", "roles/redis.serviceAgent", "roles/redis.viewer", ]) - role = each.key + role = each.value member = "serviceAccount:${google_service_account.playground_service_account.email}" project = var.project_id } + +resource "google_project_iam_member" "cloudfunction" { + for_each = toset([ + "roles/storage.objectViewer","roles/cloudfunctions.invoker","roles/datastore.user", + ]) + role = each.key + member = "serviceAccount:${google_service_account.playground_service_account_cf.email}" + project = var.project_id +} diff --git a/playground/terraform/infrastructure/setup/output.tf b/playground/terraform/infrastructure/setup/output.tf index 547bc4fe977..c4f74e6d986 100644 --- a/playground/terraform/infrastructure/setup/output.tf +++ b/playground/terraform/infrastructure/setup/output.tf @@ -19,4 +19,8 @@ output "service_account_email" { value = google_service_account.playground_service_account.email -} \ No newline at end of file +} + +output "service_account_email_cf" { + value = google_service_account.playground_service_account_cf.email +} diff --git a/playground/terraform/infrastructure/variables.tf b/playground/terraform/infrastructure/variables.tf index 80fddf807c9..f3f9efe75ca 100644 --- a/playground/terraform/infrastructure/variables.tf +++ b/playground/terraform/infrastructure/variables.tf @@ -31,6 +31,10 @@ variable "region" { description = "Infrastructure Region" } +variable "env" {} + +variable "state_bucket" {} + #IAM variable "service_account_id" { @@ -129,7 +133,7 @@ variable "network_region" { default = "us-central1" } -variable "ip-address-name" { +variable "ip_address_name" { description = "Static IP address name" default = "pg-static-ip" } diff --git a/playground/terraform/main.tf b/playground/terraform/main.tf index e852ba9ffc6..a91e875736a 100644 --- a/playground/terraform/main.tf +++ b/playground/terraform/main.tf @@ -20,12 +20,14 @@ module "infrastructure" { source = "./infrastructure" project_id = var.project_id - environment = var.environment region = var.region + environment = var.environment network_region = var.region redis_region = var.region location = var.zone service_account_id = var.service_account_id + state_bucket = var.state_bucket + env = var.env #Artifact Registry repository_id = var.repository_id repository_location = var.region @@ -37,7 +39,7 @@ module "infrastructure" { redis_memory_size_gb = var.redis_memory_size_gb #NETWORK network_name = var.network_name - ip-address-name = var.ip-address-name + ip_address_name = var.ip_address_name subnetwork_name = var.subnetwork_name #GKE gke_machine_type = var.gke_machine_type diff --git a/playground/terraform/output.tf b/playground/terraform/output.tf index d60af591eca..38f1859219e 100644 --- a/playground/terraform/output.tf +++ b/playground/terraform/output.tf @@ -63,4 +63,16 @@ output "playground_gke_project" { output "playground_static_ip_address_name" { value = module.infrastructure.playground_static_ip_address_name -} \ No newline at end of file +} + +output "playground_function_cleanup_url" { + value = module.infrastructure.playground_function_cleanup_url +} + +output "playground_function_put_url" { + value = module.infrastructure.playground_function_put_url +} + +output "playground_function_view_url" { + value = module.infrastructure.playground_function_view_url +} diff --git a/playground/terraform/variables.tf b/playground/terraform/variables.tf index a8b3c9144ab..13d467aac51 100644 --- a/playground/terraform/variables.tf +++ b/playground/terraform/variables.tf @@ -27,6 +27,10 @@ variable "environment" { description = "prod,dev,beta" } +variable "env" { + description = "prod,dev,beta" +} + variable "region" { description = "Infrastructure Region" } @@ -35,6 +39,7 @@ variable "zone" { description = "Infrastructure Zone" } +variable "state_bucket" {} # Infrastructure variables #GKE @@ -77,7 +82,7 @@ variable "service_account_id" { } #Network -variable "ip-address-name" { +variable "ip_address_name" { description = "Static IP address name" default = "pg-static-ip" }