AydarZaynutdinov commented on a change in pull request #16484:
URL: https://github.com/apache/beam/pull/16484#discussion_r789675315
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -294,3 +299,29 @@ func (controller *playgroundController)
GetPrecompiledObjectLogs(ctx context.Con
response := pb.GetPrecompiledObjectLogsResponse{Output: logs}
return &response, nil
}
+
+// GetDefaultPrecompiledObject returns the default precompile object for sdk.
+func (controller *playgroundController) GetDefaultPrecompiledObject(ctx
context.Context, info *pb.GetDefaultPrecompiledObjectRequest)
(*pb.GetDefaultPrecompiledObjectResponse, error) {
+ switch info.Sdk {
+ case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_SCIO:
Review comment:
I guess we can include SCIO as a supported SDK.
##########
File path: playground/backend/internal/utils/precompiled_objects_utils.go
##########
@@ -34,7 +43,61 @@ func PutPrecompiledObjectsToCategory(categoryName string,
precompiledObjects *cl
Type: object.Type,
PipelineOptions: object.PipelineOptions,
Link: object.Link,
+ DefaultExample: object.DefaultExample,
})
}
sdkCategory.Categories = append(sdkCategory.Categories, &category)
}
+
+// GetPrecompiledObjectsCatalogFromCache returns the precompiled objects
catalog from the cache
+func GetPrecompiledObjectsCatalogFromCache(ctx context.Context, cacheService
cache.Cache) ([]*pb.Categories, error) {
Review comment:
Lets rename it to `GetCatalogFromCache`
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -241,28 +241,33 @@ func (controller *playgroundController) Cancel(ctx
context.Context, info *pb.Can
}
// GetPrecompiledObjects returns the list of examples
+// - If SDK and category are unspecified in the request, gets the whole
catalog from the cache
+// - If there is no catalog in the cache, gets the catalog from the
Storage and saves it to the cache
+// - If SDK or category is specified in the request, gets the specific catalog
from the Storage
Review comment:
Why do we get from the cache only if SDK and category are unspecified?
Maybe it would be better to get from the cache for each request and if cache is
empty or doesn't contain required catalog/SDK objects get them from the bucket
and also save them to the cache.
What do you think?
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -294,3 +299,29 @@ func (controller *playgroundController)
GetPrecompiledObjectLogs(ctx context.Con
response := pb.GetPrecompiledObjectLogsResponse{Output: logs}
return &response, nil
}
+
+// GetDefaultPrecompiledObject returns the default precompile object for sdk.
+func (controller *playgroundController) GetDefaultPrecompiledObject(ctx
context.Context, info *pb.GetDefaultPrecompiledObjectRequest)
(*pb.GetDefaultPrecompiledObjectResponse, error) {
+ switch info.Sdk {
+ case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_SCIO:
+ logger.Errorf("GetDefaultPrecompiledObject(): unimplemented
sdk: %s\n", info.Sdk)
+ return nil, errors.InvalidArgumentError("Error during
preparing", "Sdk is not implemented yet: %s", info.Sdk.String())
+ }
+
+ defaultPrecompiledObjects, err := controller.cacheService.GetValue(ctx,
uuid.Nil, cache.DefaultPrecompiledObjects)
Review comment:
Is it correct that now we have the next approach to keep precompiled
objects into the cache:
```
nil uuid:
EXAMPLES_CATALOG:
some precompiled objects grouped by SDK.
DEFAULT_PRECOMPILED_OBJECTS
some precompiled objects grouped by SDK which also contain default:
true value into the tag.
```
In that case, the cache contains some objects (`WordCount` for example) for
`EXAMPLES_CATALOG` and for `DEFAULT_PRECOMPILED_OBJECTS`.
Maybe we can keep all precompiled objects into one subkey `EXAMPLES_CATALOG`
and for `GetDefaultPrecompiledObject()` method receive all objects from the
cache and filter them before sending them to the client.
##########
File path: playground/backend/internal/utils/precompiled_objects_utils.go
##########
@@ -34,7 +43,61 @@ func PutPrecompiledObjectsToCategory(categoryName string,
precompiledObjects *cl
Type: object.Type,
PipelineOptions: object.PipelineOptions,
Link: object.Link,
+ DefaultExample: object.DefaultExample,
})
}
sdkCategory.Categories = append(sdkCategory.Categories, &category)
}
+
+// GetPrecompiledObjectsCatalogFromCache returns the precompiled objects
catalog from the cache
+func GetPrecompiledObjectsCatalogFromCache(ctx context.Context, cacheService
cache.Cache) ([]*pb.Categories, error) {
+ value, err := cacheService.GetValue(ctx, ExamplesDataPipelineId,
cache.ExamplesCatalog)
+ if err != nil {
+ logger.Errorf("%s: cache.GetValue: %s\n",
ExamplesDataPipelineId, err.Error())
+ return nil, err
+ }
+ catalog, converted := value.([]*pb.Categories)
Review comment:
Did you test this part with Redis?
##########
File path: playground/backend/internal/utils/precompiled_objects_utils.go
##########
@@ -34,7 +43,61 @@ func PutPrecompiledObjectsToCategory(categoryName string,
precompiledObjects *cl
Type: object.Type,
PipelineOptions: object.PipelineOptions,
Link: object.Link,
+ DefaultExample: object.DefaultExample,
})
}
sdkCategory.Categories = append(sdkCategory.Categories, &category)
}
+
+// GetPrecompiledObjectsCatalogFromCache returns the precompiled objects
catalog from the cache
+func GetPrecompiledObjectsCatalogFromCache(ctx context.Context, cacheService
cache.Cache) ([]*pb.Categories, error) {
+ value, err := cacheService.GetValue(ctx, ExamplesDataPipelineId,
cache.ExamplesCatalog)
+ if err != nil {
+ logger.Errorf("%s: cache.GetValue: %s\n",
ExamplesDataPipelineId, err.Error())
+ return nil, err
+ }
+ catalog, converted := value.([]*pb.Categories)
+ if !converted {
+ logger.Errorf("%s: couldn't convert value to catalog: %s",
cache.ExamplesCatalog, value)
+ return nil, errors.InternalError("Error during getting the
catalog from cache", "Error during getting catalog")
+ }
+ return catalog, nil
+}
+
+// GetPrecompiledObjectsCatalogFromStorage returns the precompiled objects
catalog from the cloud storage
+func GetPrecompiledObjectsCatalogFromStorage(ctx context.Context, sdk pb.Sdk,
category string) ([]*pb.Categories, error) {
Review comment:
Lets rename it to `GetCatalogFromStorage`
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -178,10 +178,12 @@ func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error)
{
sdk = pb.Sdk_SDK_PYTHON
case pb.Sdk_SDK_SCIO.String():
sdk = pb.Sdk_SDK_SCIO
+ default:
+ return nil, errors.New("incorrect value of sdk in the
environment")
}
}
if sdk == pb.Sdk_SDK_UNSPECIFIED {
- return nil, errors.New("env BEAM_SDK must be specified in the
environment variables")
+ return NewBeamEnvs(sdk, nil, preparedModDir,
numOfParallelJobs), nil
Review comment:
It seems that if SDK is Unspecified then we receive `nil,
errors.New("incorrect value of SDK in the environment")` on the `182` line, so
code on the `185` line doesn't require.
Maybe we need to remove `default` case?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
