http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/DefaultMetadataQueryBuilder.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/DefaultMetadataQueryBuilder.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/DefaultMetadataQueryBuilder.java new file mode 100755 index 0000000..c9e59e7 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/DefaultMetadataQueryBuilder.java @@ -0,0 +1,69 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +public class DefaultMetadataQueryBuilder extends MetadataQueryBuilder { + + public static final String SEPARATOR_STRING = " "; + public static final String DATASET_IDENTIFIER = "from"; + public static final String CONDITION_PREFIX = "where"; + public static final String AND_IDENTIFIER = "and"; + public static final String EQUALS_IDENTIFIER = "="; + public static final String NOT_EQUALS_IDENTIFIER = "<>"; + public static final String QUOTE_IDENTIFIER = "'"; + + @Override + public String build() { + if (this.objectType != null) { + StringBuilder query = new StringBuilder(DATASET_IDENTIFIER + SEPARATOR_STRING + objectType); + if (this.conditions != null) { + boolean firstCondition = true; + for (Condition condition : conditions) { + if (condition instanceof SimpleCondition) { + SimpleCondition simpleCond = (SimpleCondition) condition; + if (firstCondition) { + query.append(SEPARATOR_STRING + AND_IDENTIFIER + SEPARATOR_STRING); + } else { + query.append(SEPARATOR_STRING + CONDITION_PREFIX + SEPARATOR_STRING); + } + query.append(simpleCond.getAttributeName()); + switch (simpleCond.getComparator()) { + case EQUALS: + query.append(SEPARATOR_STRING + EQUALS_IDENTIFIER + SEPARATOR_STRING); + break; + case NOT_EQUALS: + query.append(SEPARATOR_STRING + NOT_EQUALS_IDENTIFIER + SEPARATOR_STRING); + break; + default: + throw new RuntimeException("Comparator " + simpleCond.getComparator() + " is currently not supported"); + } + Object val = simpleCond.getValue(); + if (val instanceof MetaDataObjectReference) { + query.append(QUOTE_IDENTIFIER + ((MetaDataObjectReference) val).getId() + QUOTE_IDENTIFIER); + } else if (val instanceof String) { + query.append(QUOTE_IDENTIFIER + val.toString() + QUOTE_IDENTIFIER); + } else if (val == null) { + query.append("null"); + } else { + query.append(val.toString()); + } + } + firstCondition = false; + } + } + return query.toString(); + } + return null; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ExternalStore.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ExternalStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ExternalStore.java new file mode 100755 index 0000000..41ad9e1 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ExternalStore.java @@ -0,0 +1,44 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.Properties; + +/** + * A common interface for stores that are external to ODF. + * Provides connection test methods and basic metadata about the store. + * + * + */ +public interface ExternalStore { + static enum ConnectionStatus { OK, AUTHORIZATION_FAILED, UNREACHABLE, UNKOWN_ERROR }; + + static final String STORE_PROPERTY_DESCRIPTION = "STORE_PROPERTY_DESCRIPTION"; + static final String STORE_PROPERTY_TYPE = "STORE_PROPERTY_TYPE"; + static final String STORE_PROPERTY_ID = "STORE_PROPERTY_ID"; + + /** + * @return the properties of this metadata object store instance. + * Must return at least STORE_PROPERTY_DESCRIPTION, STORE_PROPERTY_TYPE, and STORE_PROPERTY_ID. + */ + Properties getProperties(); + + /** + * @return the unique repository Id for this metadata store + */ + String getRepositoryId(); + + ConnectionStatus testConnection(); + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetaDataUtils.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetaDataUtils.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetaDataUtils.java new file mode 100755 index 0000000..7eca5cb --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetaDataUtils.java @@ -0,0 +1,88 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; + +/** + * Internal metadata utilities + * + */ +public class InternalMetaDataUtils { + public static final String ODF_PARENT_REFERENCE = "PARENT"; + public static final String ODF_CHILDREN_REFERENCE = "CHILDREN"; + + /** + * Turn a list of metadata objects into a list of references to the corresponding metadata objects + * + * @param objectList Given list of metadata objects + * @return Resulting list of references to the metadata objects + */ + public static List<MetaDataObjectReference> getReferenceList(List<MetaDataObject> objectList) { + List<MetaDataObjectReference> result = new ArrayList<MetaDataObjectReference>(); + for (MetaDataObject obj : objectList) { + result.add(obj.getReference()); + } + return result; + } + + /** + * Convert a list of metadata object references into a list of the corresponding metadata objects + * + * @param referenceList Given list of metadata object references + * @return Resulting list metadata objects + */ + public static <T> List<T> getObjectList(MetadataStore mds, List<MetaDataObjectReference> referenceList, Class<T> type) { + List<T> result = new ArrayList<T>(); + for (MetaDataObjectReference ref : referenceList) { + MetaDataObject obj = mds.retrieve(ref); + if (obj != null) { + try { + result.add(type.cast(obj)); + } catch(ClassCastException e) { + String errorMessage = MessageFormat.format("Metadata object with id ''{0}'' cannot be cast to type ''{1}''.", new Object[] { ref.getId(), type.getName() }); + throw new MetadataStoreException(errorMessage); + } + } else { + String errorMessage = MessageFormat.format("Metadata object with reference ''{0}'' could not be retrieved from metadata store ''{1}''.", new Object[] { ref, mds.getRepositoryId() }); + throw new MetadataStoreException(errorMessage); + } + } + return result; + } + + /** + * Merge a set of given list of references to metadata objects into a single list. + * + * @param refListArray Array of given lists of references + * @return Resulting merged list of references + */ + @SafeVarargs + public static List<MetaDataObjectReference> mergeReferenceLists(List<MetaDataObjectReference>... refListArray) { + HashMap<String, MetaDataObjectReference> referenceHashMap = new HashMap<String, MetaDataObjectReference>(); + for (List<MetaDataObjectReference> refList : refListArray) { + if (refList != null) { + for (MetaDataObjectReference ref : refList) { + referenceHashMap.put(ref.getId(), ref); + } + } + } + return new ArrayList<MetaDataObjectReference>(referenceHashMap.values()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetadataStoreBase.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetadataStoreBase.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetadataStoreBase.java new file mode 100755 index 0000000..e5ebfda --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InternalMetadataStoreBase.java @@ -0,0 +1,93 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; + +/** + * Common base for default metadata store and metadata cache. + * + * + */ +public abstract class InternalMetadataStoreBase extends MetadataStoreBase implements MetadataStore { + + protected abstract HashMap<String, StoredMetaDataObject> getObjects(); + + protected <T> List<T> getReferences(String attributeName, MetaDataObject metaDataObject, Class<T> type) { + if ((metaDataObject == null) || (metaDataObject.getReference() == null)) { + throw new MetadataStoreException("Metadata object or its reference attribute cannot be null."); + } + List<T> result = new ArrayList<T>(); + StoredMetaDataObject internalObj = getObjects().get(metaDataObject.getReference().getId()); + if ((internalObj != null) && (internalObj.getReferenceMap().get(attributeName) != null)) { + for (MetaDataObjectReference ref : internalObj.getReferenceMap().get(attributeName)) { + MetaDataObject obj = retrieve(ref); + if (obj != null) { + // Ignore objects that are not available in metadata store + // TODO: Consider to use invalide reference if an object is not available + try { + result.add(type.cast(retrieve(ref))); + } catch(ClassCastException e) { + String errorMessage = MessageFormat.format("Inconsistent object reference: A reference of type ''{0}'' cannot be cast to type ''{1}''.", new Object[] { attributeName, type.getName() }); + throw new MetadataStoreException(errorMessage); + } + } + } + } + return result; + } + + abstract protected Object getAccessLock(); + + @Override + public MetaDataObject getParent(MetaDataObject metaDataObject) { + List<MetaDataObject> parentList = new ArrayList<MetaDataObject>(); + // TODO: Make this more efficient + for (StoredMetaDataObject internalMdo : getObjects().values()) { + for (MetaDataObject child : getChildren(internalMdo.getMetaDataObject())) { + if (child.getReference().getId().equals(metaDataObject.getReference().getId())) { + parentList.add(internalMdo.getMetaDataObject()); + } + } + } + if (parentList.size() == 1) { + return parentList.get(0); + } else if (parentList.size() == 0) { + return null; + } + String errorMessage = MessageFormat.format("Inconsistent object reference: Metadata object with id ''{0}'' refers to more that one parent object.", metaDataObject.getReference().getId()); + throw new MetadataStoreException(errorMessage); + } + + @Override + public MetaDataObject retrieve(MetaDataObjectReference reference) { + synchronized(getAccessLock()) { + String objectId = reference.getId(); + if (getObjects().containsKey(objectId)) { + return getObjects().get(objectId).getMetaDataObject(); + } + return null; + } + } + + @Override + public MetadataQueryBuilder newQueryBuilder() { + return new DefaultMetadataQueryBuilder(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InvalidReference.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InvalidReference.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InvalidReference.java new file mode 100755 index 0000000..d112720 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/InvalidReference.java @@ -0,0 +1,77 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.ArrayList; +import java.util.List; + +/** + * Helper method to handle "invalid" references. + * + * Invalid references are typically returned by the metadata store implementation to indicate that a reference (or a reference list) was not provided. + * This could be the case, e.g., for performance reasons when finding a reference might be time consuming. + * In such a case the application should explicitly use the MetadataQueryBuilder to get to the reference (list). + * + * Clients should check any MetadataObjectReference and List<MetadataObjectRefernce> + * in retrieved MetadataObjects if the value is an instance of this class. + * + * + */ +public class InvalidReference { + + public static final String INVALID_METADATAOBJECT_REFERENCE_ID = "INVALID_METADATAOBJECT_REFERENCE_ID"; + public static final String INVALID_METADATAOBJECT_REFERENCE_LIST_ID = "INVALID_METADATAOBJECT_REFERENCE_LIST_ID"; + + /** + * use this method to indicate that a reference is invalid. + */ + public static MetaDataObjectReference createInvalidReference(String repositoryId) { + MetaDataObjectReference invalidRef = new MetaDataObjectReference(); + invalidRef.setRepositoryId(repositoryId); + invalidRef.setId(INVALID_METADATAOBJECT_REFERENCE_ID); + return invalidRef; + } + + public static boolean isInvalidRef(MetaDataObjectReference ref) { + if (ref == null) { + return false; + } + return INVALID_METADATAOBJECT_REFERENCE_ID.equals(ref.getId()); + } + + + /** + * use this method to indicate that a list of references is invalid. + */ + public static List<MetaDataObjectReference> createInvalidReferenceList(String repositoryId) { + List<MetaDataObjectReference> invalidRefList = new ArrayList<>(); + MetaDataObjectReference invalidRefMarker = new MetaDataObjectReference(); + invalidRefMarker.setRepositoryId(repositoryId); + invalidRefMarker.setId(INVALID_METADATAOBJECT_REFERENCE_LIST_ID); + invalidRefList.add(invalidRefMarker); + return invalidRefList; + } + + public static boolean isInvalidRefList(List<MetaDataObjectReference> refList) { + if (refList.size() != 1) { + return false; + } + MetaDataObjectReference ref = refList.get(0); + if (ref == null) { + return false; + } + return INVALID_METADATAOBJECT_REFERENCE_LIST_ID.equals(ref.getId()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetaDataObjectReference.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetaDataObjectReference.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetaDataObjectReference.java new file mode 100755 index 0000000..de61568 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetaDataObjectReference.java @@ -0,0 +1,100 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +// JSON +/** + * This class describes the location of a MetadataObject + * + */ +@ApiModel(description="Reference to a metadata object.") +public class MetaDataObjectReference { + @ApiModelProperty(value="Unique id of the object", required=true) + private String id; + + @ApiModelProperty(value="Id of the metadata repository where the object is registered", required=true) + private String repositoryId; + + @ApiModelProperty(value="URL of the object in the metadata repository", required=true) + private String url; + + @JsonIgnore + private ReferenceCache cache; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (!(other instanceof MetaDataObjectReference)) { + return false; + } + MetaDataObjectReference otherMDO = (MetaDataObjectReference) other; + if (!this.id.equals(otherMDO.id)) { + return false; + } + if (this.repositoryId == null) { + return otherMDO.repositoryId == null; + } + return this.repositoryId.equals(otherMDO.repositoryId); + } + + public int hashCode() { + int result = 0; + if (this.repositoryId != null) { + result = repositoryId.hashCode(); + } + return result + this.id.hashCode(); + } + + public String toString() { + return this.repositoryId + "|||" + this.id; + } + + public String getRepositoryId() { + return repositoryId; + } + + public void setRepositoryId(String repositoryId) { + this.repositoryId = repositoryId; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public ReferenceCache getCache() { + return cache; + } + + public void setCache(ReferenceCache cache) { + this.cache = cache; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataQueryBuilder.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataQueryBuilder.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataQueryBuilder.java new file mode 100755 index 0000000..643f203 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataQueryBuilder.java @@ -0,0 +1,92 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.ArrayList; +import java.util.List; + +/** + * Abstract base class for a builder that can be used to create metadata queries. + * It uses the Java builder pattern. + * + * There are two types of methods: + * 1. Chainable methods that can be used to do simple filtering, e.g., + * {@code String query = queryBuilder.objectType("DataSet").simpleCondition("name", COMPARATOR.EQUALS, "waldo").build();} + * 2. Predefined queries that are not chainable. These are very specific queries that currently cannot be built with the chainable methods, e.g., + * {@code String query = queryBuilder.connectionsForDataSet(dataSetId).build();} + * + * When subclassing, note that the methods set the appropriate protected fields to null to indicate that the query was "overwritten". + * + * @See {@link MetadataStore} + */ +public abstract class MetadataQueryBuilder { + + public static enum COMPARATOR { + EQUALS, NOT_EQUALS + }; + + protected static class Condition { + }; + + protected static class SimpleCondition extends Condition { + public SimpleCondition(String attributeName, COMPARATOR comparator, Object value) { + super(); + this.attributeName = attributeName; + this.comparator = comparator; + this.value = value; + } + + private String attributeName; + private COMPARATOR comparator; + private Object value; + + public String getAttributeName() { + return attributeName; + } + + public COMPARATOR getComparator() { + return comparator; + } + + public Object getValue() { + return value; + } + + } + + protected String objectType; + protected List<Condition> conditions; + + public abstract String build(); + + /** + * Set the type of object to be queried. Names are the ones of the common model (e.g. Table, Column, etc.) + */ + public MetadataQueryBuilder objectType(String objectTypeName) { + this.objectType = objectTypeName; + return this; + } + + /** + * Add a simple condition to the query. All conditions are "ANDed". + */ + public MetadataQueryBuilder simpleCondition(String attributeName, COMPARATOR comparator, Object value) { + if (conditions == null) { + conditions = new ArrayList<>(); + } + conditions.add(new SimpleCondition(attributeName, comparator, value)); + return this; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStore.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStore.java new file mode 100755 index 0000000..7a50ced --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStore.java @@ -0,0 +1,173 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.List; + +import org.apache.atlas.odf.api.metadata.models.Column; +import org.apache.atlas.odf.api.metadata.models.Connection; +import org.apache.atlas.odf.api.metadata.models.ConnectionInfo; +import org.apache.atlas.odf.api.metadata.models.DataFile; +import org.apache.atlas.odf.api.metadata.models.DataFileFolder; +import org.apache.atlas.odf.api.metadata.models.DataStore; +import org.apache.atlas.odf.api.metadata.models.Database; +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; +import org.apache.atlas.odf.api.metadata.models.RelationalDataSet; +import org.apache.atlas.odf.api.metadata.models.Schema; +import org.apache.atlas.odf.api.metadata.models.Table; + +/** + * Interfaces to be implemented by a metadata store in order to be used with ODF. + * + * In addition to this interface, each ODF metadata store must support the ODF base types defined by the + * {@link WritableMetadataStoreUtils#getBaseTypes} method. + * + */ +public interface MetadataStore extends ExternalStore { + + /** + * Retrieve information required to access the actual data behind an information asset, e.g. the connection info + * to retrieve the data in a JDBC table. + * + * @param informationAsset Given information asset + * @return Connection information required for data access + */ + ConnectionInfo getConnectionInfo(MetaDataObject informationAsset); + + /** + * Retrieve a metadata object by its metadata object reference. + * + * @param reference Metadata object reference + * @return Metadata object + */ + MetaDataObject retrieve(MetaDataObjectReference reference); + + /** + * Perform a search against the metadata store. The query should be generated using the {@link MetadataQueryBuilder} + * returned by the {@link #newQueryBuilder()} method. + + * @param query Query string + * @return List of references to metadata objects found by the query + */ + List<MetaDataObjectReference> search(String query); + + /** + * Populates the metadata store with example datasets. This method is optional, however in order to support the ODF + * integration tests, this method must create the object returned by the {@link WritableMetadataStoreUtils#getSampleDataObjects} + * method. + * + */ + void createSampleData(); + + /** + * Deletes all data from this repository. This method is optional, however it must be implemented in order to support the ODF + * integration tests. + * + */ + void resetAllData(); + + MetadataQueryBuilder newQueryBuilder(); + + /** + * Return an implementation of the {@link AnnotationPropagator} interface that propagates ODF annotations into the metadata store. + * The method may return null if the metadata store does not support annotation propagation. + * + * return the AnnotationPropagator for this MetadataStore. + */ + AnnotationPropagator getAnnotationPropagator(); + + /** + * Retrieve references of a specific type from an object stored in the metadata store. + * A list of available reference types can be retrieved with the {@link #getReferenceTypes() getReferenceTypes} method. + * + * @param metaDataObject Given metadata object to retrieve the references from + * @param attributeName Name of the reference + * @return List of objects referenced by the given metadata object + */ + public List<MetaDataObject> getReferences(String attributeName, MetaDataObject metaDataObject); + + /** + * Return the list of available reference types supported by the {@link #getReferences(String, MetaDataObject) getReferences} method of the metadata store. + * The list indicates which reference types are added to the internal metadata cache when a discovery service is called. That way, they will be available + * to the service at runtime even if the service has no access to the metadata store. + * + * @return List of supported reference types + */ + public List<String> getReferenceTypes(); + + /** + * Retrieve the parent object of a given object stored in the metadata store. + * + * @param metaDataObject Given metadata object + * @return Parent object of the metadata object + */ + public MetaDataObject getParent(MetaDataObject metaDataObject); + + /** + * Retrieve the child objects of a given object stored in the metadata store. + * + * @param metaDataObject Given metadata object + * @return List of child objects objects referenced by the given metadata object + */ + public List<MetaDataObject> getChildren(MetaDataObject metaDataObject); + + /** + * Retrieve data file objects referenced by a data file folder object. + * + * @param metaDataObject Given metadata object + * @return List of data file objects + */ + public List<DataFile> getDataFiles(DataFileFolder folder); + + /** + * Retrieve data file folder objects referenced by a data file folder object. + * + * @param metaDataObject Given metadata object + * @return List of data file folder objects + */ + public List<DataFileFolder> getDataFileFolders(DataFileFolder folder); + + /** + * Retrieve schema objects referenced by a database object. + * + * @param metaDataObject Given metadata object + * @return List of schema objects + */ + public List<Schema> getSchemas(Database database); + + /** + * Retrieve table objects referenced by a schema object. + * + * @param metaDataObject Given metadata object + * @return List of table objects + */ + public List<Table> getTables(Schema schema); + + /** + * Retrieve column objects referenced by a table object. + * + * @param metaDataObject Given metadata object + * @return List of column objects + */ + public List<Column> getColumns(RelationalDataSet relationalDataSet); + + /** + * Retrieve connection objects referenced by a data store object. + * + * @param metaDataObject Given metadata object + * @return List of connection objects + */ + public List<Connection> getConnections(DataStore dataStore); + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreBase.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreBase.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreBase.java new file mode 100755 index 0000000..9ad68bf --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreBase.java @@ -0,0 +1,111 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.atlas.odf.api.metadata.models.Column; +import org.apache.atlas.odf.api.metadata.models.Connection; +import org.apache.atlas.odf.api.metadata.models.DataFile; +import org.apache.atlas.odf.api.metadata.models.DataFileFolder; +import org.apache.atlas.odf.api.metadata.models.DataStore; +import org.apache.atlas.odf.api.metadata.models.Database; +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; +import org.apache.atlas.odf.api.metadata.models.RelationalDataSet; +import org.apache.atlas.odf.api.metadata.models.Schema; +import org.apache.atlas.odf.api.metadata.models.Table; + +/** + * Common base that may be used for any metadata store implementation. + * + * + */ +public abstract class MetadataStoreBase implements MetadataStore { + + public static final String ODF_CONNECTIONS_REFERENCE = "CONNECTIONS"; + public static final String ODF_COLUMNS_REFERENCE = "COLUMNS"; + public static final String ODF_DATAFILEFOLDERS_REFERENCE = "DATAFILEFOLDERS"; + public static final String ODF_DATAFILES_REFERENCE = "DATAFILES"; + public static final String ODF_SCHEMAS_REFERENCE = "SCHEMAS"; + public static final String ODF_TABLES_REFERENCE = "TABLES"; + + protected abstract <T> List<T> getReferences(String attributeName, MetaDataObject metaDataObject, Class<T> type); + + @Override + public List<String> getReferenceTypes() { + List<String> result = new ArrayList<String>(); + result.add(ODF_CONNECTIONS_REFERENCE); + result.add(ODF_COLUMNS_REFERENCE); + result.add(ODF_DATAFILEFOLDERS_REFERENCE); + result.add(ODF_DATAFILES_REFERENCE); + result.add(ODF_SCHEMAS_REFERENCE); + result.add(ODF_TABLES_REFERENCE); + return result; + } + + @Override + public List<MetaDataObject> getReferences(String attributeName, MetaDataObject metaDataObject) { + return getReferences(attributeName, metaDataObject, MetaDataObject.class); + } + + @Override + public List<DataFile> getDataFiles(DataFileFolder folder) { + return getReferences(ODF_DATAFILES_REFERENCE, folder, DataFile.class); + } + + @Override + public List<DataFileFolder> getDataFileFolders(DataFileFolder folder) { + return getReferences(ODF_DATAFILEFOLDERS_REFERENCE, folder, DataFileFolder.class); + } + + @Override + public List<Schema> getSchemas(Database database) { + return getReferences(ODF_SCHEMAS_REFERENCE, database, Schema.class); + } + + @Override + public List<Table> getTables(Schema schema) { + return getReferences(ODF_TABLES_REFERENCE, schema, Table.class); + } + + @Override + public List<Column> getColumns(RelationalDataSet relationalDataSet) { + return getReferences(ODF_COLUMNS_REFERENCE, relationalDataSet, Column.class); + } + + @Override + public List<Connection> getConnections(DataStore dataStore) { + return getReferences(ODF_CONNECTIONS_REFERENCE, dataStore, Connection.class); + } + + @Override + public ConnectionStatus testConnection() { + return ConnectionStatus.OK; + } + + @Override + public List<MetaDataObject> getChildren(MetaDataObject metaDataObject) { + List<MetaDataObject> result = new ArrayList<MetaDataObject>(); + for (String referenceType : getReferenceTypes()) { + for (MetaDataObject ref : getReferences(referenceType, metaDataObject, MetaDataObject.class)) { + if (!result.contains(ref)) { + result.add(ref); + } + } + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreException.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreException.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreException.java new file mode 100755 index 0000000..7c84a61 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/MetadataStoreException.java @@ -0,0 +1,36 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +public class MetadataStoreException extends RuntimeException { + + private static final long serialVersionUID = -8509622412001869582L; + + public MetadataStoreException() { + super(); + } + + public MetadataStoreException(String message, Throwable cause) { + super(message, cause); + } + + public MetadataStoreException(String message) { + super(message); + } + + public MetadataStoreException(Throwable cause) { + super(cause); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RESTMetadataStoreHelper.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RESTMetadataStoreHelper.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RESTMetadataStoreHelper.java new file mode 100755 index 0000000..5601614 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RESTMetadataStoreHelper.java @@ -0,0 +1,51 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.atlas.odf.api.connectivity.RESTClientManager; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.fluent.Response; + +public class RESTMetadataStoreHelper { + + static Logger logger = Logger.getLogger(RESTMetadataStoreHelper.class.getName()); + + /** + * Return a ConnectionStatus object assuming that the URI is static in the sense that + * the metadata store is unreachable if the URI cannot be reached. + */ + public static MetadataStore.ConnectionStatus testConnectionForStaticURL(RESTClientManager client, String uri) { + try { + Response resp = client.getAuthenticatedExecutor().execute(Request.Get(uri)); + HttpResponse httpResponse = resp.returnResponse(); + switch (httpResponse.getStatusLine().getStatusCode()) { + case HttpStatus.SC_NOT_FOUND: + return MetadataStore.ConnectionStatus.UNREACHABLE; + case HttpStatus.SC_OK: + return MetadataStore.ConnectionStatus.OK; + default: + ; + } + } catch (Exception e) { + logger.log(Level.INFO, "Connection failed", e); + } + return MetadataStore.ConnectionStatus.UNKOWN_ERROR; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ReferenceCache.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ReferenceCache.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ReferenceCache.java new file mode 100755 index 0000000..d48b6fe --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/ReferenceCache.java @@ -0,0 +1,54 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import org.apache.atlas.odf.api.metadata.models.Annotation; +import org.apache.atlas.odf.api.metadata.models.Column; +import org.apache.atlas.odf.api.metadata.models.DataSet; + +/** + * This class is used to cache the materialized version of a metadata reference, in order to reduce the number of retrievals required + * + */ +public class ReferenceCache { + + private Annotation annotation; + private Column oMColumn; + private DataSet oMDataSet; + + public Column getColumn() { + return oMColumn; + } + + public void setColumn(Column oMColumn) { + this.oMColumn = oMColumn; + } + + public DataSet getDataSet() { + return oMDataSet; + } + + public void setDataSet(DataSet oMDataSet) { + this.oMDataSet = oMDataSet; + } + + public Annotation getAnnotation() { + return annotation; + } + + public void setAnnotation(Annotation annotation) { + this.annotation = annotation; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RemoteMetadataStore.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RemoteMetadataStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RemoteMetadataStore.java new file mode 100755 index 0000000..3567c1d --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/RemoteMetadataStore.java @@ -0,0 +1,385 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.security.GeneralSecurityException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Hashtable; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.utils.URIBuilder; +import org.apache.wink.json4j.JSON; +import org.apache.wink.json4j.JSONArray; +import org.apache.wink.json4j.JSONException; +import org.apache.wink.json4j.JSONObject; + +import org.apache.atlas.odf.api.connectivity.RESTClientManager; +import org.apache.atlas.odf.api.metadata.models.ConnectionInfo; +import org.apache.atlas.odf.json.JSONUtils; + +// TODO properly escape all URLs when constructed as string concatenation + +/** + * + * A MetadataStore to access metadata via an ODF instance + * + */ +public class RemoteMetadataStore extends MetadataStoreBase implements MetadataStore { + private Logger logger = Logger.getLogger(RemoteMetadataStore.class.getName()); + + private String odfUrl; + private String odfUser; + + private Properties mdsProps = null; + + // if this is true, null repository Ids are ok for all MetaDataObjectReference objects + private boolean isDefaultStore = true; + + private RESTClientManager restClient; + + static String ODF_API_INFIX = "/odf/api/v1"; + + private void constructThis(String odfUrl, String odfUser, String odfPassword, boolean isDefaultStore) throws URISyntaxException { + this.odfUrl = odfUrl; + this.odfUser = odfUser; + this.restClient = new RESTClientManager(new URI(odfUrl), odfUser, odfPassword); + this.isDefaultStore = isDefaultStore; + } + + public RemoteMetadataStore(String odfUrl, String odfUser, String odfPassword, boolean isDefaultStore) throws URISyntaxException, MetadataStoreException { + constructThis(odfUrl, odfUser, odfPassword, isDefaultStore); + } + + /** + * check if the reference belongs to this repository. Throw exception if not. + */ + void checkReference(MetaDataObjectReference reference) { + if (reference == null) { + throw new MetadataStoreException("Reference cannot be null"); + } + if (reference.getRepositoryId() == null) { + if (!isDefaultStore) { + throw new MetadataStoreException("Repository ID is not set on the reference."); + } + } else { + if (!reference.getRepositoryId().equals(this.getRepositoryId())) { + throw new MetadataStoreException(MessageFormat.format("Repository ID ''{0}'' of reference does not match the one of this repository ''{1}''", + new Object[] { reference.getRepositoryId(), getRepositoryId() })); + } + } + } + + /** + * check if the ODF metadata API can be reached. Throw exception if not. + */ + private void checkConnectionToMetadataAPI() { + MetadataStore.ConnectionStatus connStatus = testConnection(); + if (connStatus.equals(MetadataStore.ConnectionStatus.UNREACHABLE)) { + throw new MetadataStoreException("Internal API for metadata store cannot be reached. Make sure that the discovery service has access to the following URL: " + odfUrl); + } else if (connStatus.equals(MetadataStore.ConnectionStatus.AUTHORIZATION_FAILED)) { + String messageDetail =""; + if (this.odfUser.isEmpty()) { + messageDetail = " Make sure to connect to the discovery service securely through https."; + //Note that ODF user id and password are only provided if the connection to the service is secure + } + throw new MetadataStoreException("Autorization failure when accessing API of internal metadata store." + messageDetail); + } + } + + @Override + public ConnectionInfo getConnectionInfo(MetaDataObject informationAsset) { + throw new UnsupportedOperationException("This method is not available in the remote implementation of the Metadata store."); + }; + + @Override + public MetaDataObject retrieve(MetaDataObjectReference reference) { + checkReference(reference); + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata/asset/" + URLEncoder.encode(JSONUtils.toJSON(reference), "UTF-8"); + logger.log(Level.FINEST, "Object reference to be retrieved ''{0}''.", reference.toString()); + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code == HttpStatus.SC_NOT_FOUND) { + return null; + } + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Retrieval of object ''{0}'' failed: HTTP request status: ''{1}'', {2}", + new Object[] { JSONUtils.toJSON(reference), statusLine.getStatusCode(), statusLine.getReasonPhrase() }); + throw new MetadataStoreException(msg); + } else { + JSONObject mdo = (JSONObject) JSON.parse(httpResponse.getEntity().getContent()); + mdo.remove("annotations"); + MetaDataObject result = JSONUtils.fromJSON(mdo.write(), MetaDataObject.class); + if (result.getReference() == null) { + // An empty JSON documents indicates that the result should be null. + result = null; + } + logger.log(Level.FINEST, "Retrieved metadata object: ''{0}''.", result); + return result; + } + } catch (GeneralSecurityException | IOException | JSONException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + + @Override + public Properties getProperties() { + if (this.mdsProps != null) { + return this.mdsProps; + } else { + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata"; + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + InputStream is = httpResponse.getEntity().getContent(); + String response = JSONUtils.getInputStreamAsString(is, "UTF-8"); + is.close(); + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Retrieval of metadata store properties at ''{3}'' failed: HTTP request status: ''{0}'', {1}, details: {2}", + new Object[] { code, statusLine.getReasonPhrase(), response, resource}); + throw new MetadataStoreException(msg); + } else { + this.mdsProps = new Properties(); + JSONObject jo = new JSONObject(response); + for (Object key : jo.keySet()) { + this.mdsProps.put((String) key, (String) jo.get(key)); + } + return this.mdsProps; + } + } catch (GeneralSecurityException | IOException | JSONException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + } + + @Override + public List<MetaDataObjectReference> search(String query) { + checkConnectionToMetadataAPI(); + try { + logger.log(Level.FINE, "Metadata search term: ''{0}''.", query); + URIBuilder uri = new URIBuilder(odfUrl + ODF_API_INFIX + "/metadata/search") + .addParameter("query", query) + .addParameter("resulttype", "references"); + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(uri.build())).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code != HttpStatus.SC_OK) { + throw new MetadataStoreException("Search request failed: " + statusLine.getStatusCode() + ", " + statusLine.getReasonPhrase()); + } + InputStream is = httpResponse.getEntity().getContent(); + JSONArray objReferencesJson = new JSONArray(is); + is.close(); + logger.log(Level.FINEST, "Metadata search response: ''{0}''.", objReferencesJson.write()); + List<MetaDataObjectReference> resultMDORs = new ArrayList<>(); + for (Object ref : objReferencesJson) { + MetaDataObjectReference objRef = JSONUtils.fromJSON(((JSONObject) ref).write(), MetaDataObjectReference.class); + resultMDORs.add(objRef); + } + return resultMDORs; + } catch (GeneralSecurityException | IOException | URISyntaxException | JSONException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting to the metadata store.", exc); + throw new MetadataStoreException(exc); + } + + } + + @Override + public String getRepositoryId() { + Hashtable<Object, Object> mdsProps = (Hashtable<Object, Object>) this.getProperties(); + if (mdsProps.get(STORE_PROPERTY_ID) != null) { + return (String) mdsProps.get(STORE_PROPERTY_ID); + } else { + throw new MetadataStoreException("Property " + STORE_PROPERTY_ID + " is missing from metadata store properties ''" + mdsProps.toString() + "''."); + } + } + + @Override + public MetadataStore.ConnectionStatus testConnection() { + return RESTMetadataStoreHelper.testConnectionForStaticURL(restClient, odfUrl); + } + + @Override + public void createSampleData() { + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata/sampledata"; + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Create sample data failed: HTTP request status: ''{1}'', {2}", + new Object[] { statusLine.getStatusCode(), statusLine.getReasonPhrase() }); + throw new MetadataStoreException(msg); + } + } catch (GeneralSecurityException | IOException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + + @Override + public void resetAllData() { + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata/resetalldata"; + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Post(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Reset all data failed: HTTP request status: ''{1}'', {2}", + new Object[] { statusLine.getStatusCode(), statusLine.getReasonPhrase() }); + throw new MetadataStoreException(msg); + } + } catch (GeneralSecurityException | IOException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + + @Override + public MetadataQueryBuilder newQueryBuilder() { + String repoType = getProperties().getProperty(STORE_PROPERTY_TYPE); + if ("atlas".equals(repoType)) { + return new AtlasMetadataQueryBuilder(); + } else if ("default".equals(repoType)) { + return new DefaultMetadataQueryBuilder(); + } + throw new RuntimeException(MessageFormat.format("No query builder exists for the repository type ''{0}''", repoType)); + } + + @Override + public AnnotationPropagator getAnnotationPropagator() { + throw new UnsupportedOperationException("This method is not available in the remote implementation of the Metadata store."); + } + + protected <T> List<T> getReferences(String attributeName, MetaDataObject metaDataObject, Class<T> type){ + String objectId = metaDataObject.getReference().getId(); + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata/asset/" + + URLEncoder.encode(JSONUtils.toJSON(metaDataObject.getReference()), "UTF-8") + + "/" + URLEncoder.encode(attributeName.toLowerCase(), "UTF-8"); + logger.log(Level.FINEST, "Retrieving references of type ''{0}'' from metadata object id ''{1}''.", new Object[] { attributeName, objectId }); + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code == HttpStatus.SC_NOT_FOUND) { + return null; + } + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Retrieving references of type ''{0}'' of object id ''{1}'' failed: HTTP request status: ''{2}'', {3}", + new Object[] { attributeName, objectId, statusLine.getStatusCode(), statusLine.getReasonPhrase() }); + throw new MetadataStoreException(msg); + } else { + InputStream is = httpResponse.getEntity().getContent(); + JSONArray objReferencesJson = new JSONArray(is); + is.close(); + logger.log(Level.FINEST, "Get references response: ''{0}''.", objReferencesJson.write()); + List<T> referencedObjects = new ArrayList<T>(); + for (Object ref : objReferencesJson) { + T obj = JSONUtils.fromJSON(((JSONObject) ref).write(), type); + referencedObjects.add(obj); + } + return referencedObjects; + } + } catch (GeneralSecurityException | IOException | JSONException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + + @Override + public List<MetaDataObject> getReferences(String attributeName, MetaDataObject metaDataObject){ + return getReferences(attributeName, metaDataObject, MetaDataObject.class); + } + + @Override + public List<String> getReferenceTypes(){ + checkConnectionToMetadataAPI(); + try { + String resource = odfUrl + ODF_API_INFIX + "/metadata/referencetypes"; + Executor executor = this.restClient.getAuthenticatedExecutor(); + HttpResponse httpResponse = executor.execute(Request.Get(resource)).returnResponse(); + StatusLine statusLine = httpResponse.getStatusLine(); + int code = statusLine.getStatusCode(); + if (code == HttpStatus.SC_NOT_FOUND) { + return null; + } + if (code != HttpStatus.SC_OK) { + String msg = MessageFormat.format("Retrieving reference type names failed: HTTP request status: ''{1}'', {2}", + new Object[] { statusLine.getStatusCode(), statusLine.getReasonPhrase() }); + throw new MetadataStoreException(msg); + } else { + InputStream is = httpResponse.getEntity().getContent(); + JSONArray objReferencesJson = new JSONArray(is); + is.close(); + logger.log(Level.FINEST, "Get reference types response: ''{0}''.", objReferencesJson.write()); + List<String> referenceTypeNames = new ArrayList<String>(); + for (Object ref : objReferencesJson) { + String obj = JSONUtils.fromJSON(((JSONObject) ref).write(), String.class); + referenceTypeNames.add(obj); + } + return referenceTypeNames; + } + } catch (GeneralSecurityException | IOException | JSONException exc) { + logger.log(Level.WARNING, "An unexpected exception ocurred while connecting the metadata store", exc); + throw new MetadataStoreException(exc); + } + } + + @Override + public MetaDataObject getParent(MetaDataObject metaDataObject){ + List<MetaDataObject> parentList = getReferences(InternalMetaDataUtils.ODF_PARENT_REFERENCE, metaDataObject, MetaDataObject.class); + if (parentList.size() == 1) { + return parentList.get(0); + } else if (parentList.size() == 0) { + return null; + } + String errorMessage = MessageFormat.format("Inconsistent object reference: Metadata object with id ''{0}'' refers to more that one parent object.", metaDataObject.getReference().getId()); + throw new MetadataStoreException(errorMessage); + } + + @Override + public List<MetaDataObject> getChildren(MetaDataObject metaDataObject){ + return getReferences(InternalMetaDataUtils.ODF_CHILDREN_REFERENCE, metaDataObject, MetaDataObject.class); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/StoredMetaDataObject.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/StoredMetaDataObject.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/StoredMetaDataObject.java new file mode 100755 index 0000000..5de5f12 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/StoredMetaDataObject.java @@ -0,0 +1,61 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import java.util.HashMap; +import java.util.List; + +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * Internal representation of a metadata object that is used by the @see MetaDataCache + * In addition to the object itself this class contains all references of the object. + * + * + */ +@ApiModel(description="Internal representation of a metadata object in the metadata cache.") +public class StoredMetaDataObject { + @ApiModelProperty(value="Actual cached metadata object", readOnly=false, required=true) + private MetaDataObject metaDataObject; + + @ApiModelProperty(value="Map of all references of the cached metadata object containing one reference list for each type of reference", readOnly=false, required=true) + private HashMap<String, List<MetaDataObjectReference>> referenceMap; + + public void setMetaDataObject(MetaDataObject metaDataObject) { + this.metaDataObject = metaDataObject; + } + + public MetaDataObject getMetaDataObject() { + return this.metaDataObject; + } + + public StoredMetaDataObject() { + } + + public StoredMetaDataObject(MetaDataObject metaDataObject) { + this.metaDataObject = metaDataObject; + this.referenceMap = new HashMap<String, List<MetaDataObjectReference>>(); + } + + public void setReferencesMap(HashMap<String, List<MetaDataObjectReference>> referenceMap) { + this.referenceMap = referenceMap; + } + + public HashMap<String, List<MetaDataObjectReference>> getReferenceMap() { + return this.referenceMap; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/UnknownMetaDataObject.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/UnknownMetaDataObject.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/UnknownMetaDataObject.java new file mode 100755 index 0000000..a6de68e --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/UnknownMetaDataObject.java @@ -0,0 +1,22 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.atlas.odf.api.metadata.models.MetaDataObject; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class UnknownMetaDataObject extends MetaDataObject { + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImportResult.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImportResult.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImportResult.java new file mode 100755 index 0000000..feaef91 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImportResult.java @@ -0,0 +1,42 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.importer; + +import java.util.List; + +public class JDBCMetadataImportResult { + private String databaseName; + private List<String> tableNames; + private String dbRef; + + public JDBCMetadataImportResult(String databaseName, String dbId, List<String> tableNames) { + super(); + this.databaseName = databaseName; + this.tableNames = tableNames; + this.dbRef = dbId; + } + + public String getDBId() { + return this.dbRef; + } + + public String getDatabaseName() { + return databaseName; + } + + public List<String> getTableNames() { + return tableNames; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImporter.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImporter.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImporter.java new file mode 100755 index 0000000..2127ce6 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/JDBCMetadataImporter.java @@ -0,0 +1,36 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.importer; + +import org.apache.atlas.odf.api.metadata.models.JDBCConnection; + +/** + * Interface of the utility that imports metadata from JDBC data sources into the ODF metadata store. + * + */ +public interface JDBCMetadataImporter { + + /** + * Import metadata of one or multiple relational tables into the ODF metadata store, along with the corresponding + * database and connection information. + * + * @param connection Connection to the JDBC data soure + * @param dbName Database name + * @param schemaPattern Database schema name or pattern + * @param tableNamePattern Table name or pattern + * @return Object containing the raw results of the import operation + */ + public JDBCMetadataImportResult importTables(JDBCConnection connection, String dbName, String schemaPattern, String tableNamePattern); + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/MetadataImportException.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/MetadataImportException.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/MetadataImportException.java new file mode 100755 index 0000000..3e5cba3 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/importer/MetadataImportException.java @@ -0,0 +1,32 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.importer; + +public class MetadataImportException extends RuntimeException { + + private static final long serialVersionUID = -3502239943338011231L; + + public MetadataImportException() { + super(); + } + + public MetadataImportException(String message) { + super(message); + } + + public MetadataImportException(Throwable cause) { + super(cause); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Annotation.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Annotation.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Annotation.java new file mode 100755 index 0000000..b25ad8b --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Annotation.java @@ -0,0 +1,61 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +// JSON +/** + * This class represents a result of a discovery service analysis on a single metadata object. + * By extending this class new annotation types for new discovery services can be created in order to provide additional information + * + */ +public abstract class Annotation extends MetaDataObject { + + private String annotationType = this.getClass().getSimpleName().replace('$', '_'); + private String analysisRun; + private String jsonProperties; + private String summary; + + public String getAnnotationType() { + return annotationType; + } + + public void setAnnotationType(String annotationType) { + this.annotationType = annotationType; + } + + public String getJsonProperties() { + return jsonProperties; + } + + public void setJsonProperties(String jsonProperties) { + this.jsonProperties = jsonProperties; + } + + public String getAnalysisRun() { + return analysisRun; + } + + public void setAnalysisRun(String analysisRun) { + this.analysisRun = analysisRun; + } + + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/BusinessTerm.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/BusinessTerm.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/BusinessTerm.java new file mode 100755 index 0000000..cbc801f --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/BusinessTerm.java @@ -0,0 +1,44 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +import java.util.List; + +public class BusinessTerm extends MetaDataObject { + + private List<String> abbreviations; + private String example; + private String usage; + + public List<String> getAbbreviations() { + return abbreviations; + } + public void setAbbreviations(List<String> abbreviations) { + this.abbreviations = abbreviations; + } + public String getExample() { + return example; + } + public void setExample(String example) { + this.example = example; + } + public String getUsage() { + return usage; + } + public void setUsage(String usage) { + this.usage = usage; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/CachedMetadataStore.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/CachedMetadataStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/CachedMetadataStore.java new file mode 100755 index 0000000..5bbf731 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/CachedMetadataStore.java @@ -0,0 +1,137 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +import java.util.HashMap; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.atlas.odf.api.metadata.InternalMetadataStoreBase; +import org.apache.atlas.odf.api.metadata.MetaDataObjectReference; +import org.apache.atlas.odf.api.metadata.MetadataStore; +import org.apache.atlas.odf.api.metadata.AnnotationPropagator; +import org.apache.atlas.odf.api.metadata.InternalMetaDataUtils; +import org.apache.atlas.odf.api.metadata.StoredMetaDataObject; + +/** + * In-memory metadata cache to be used by discovery services that do not have access to the metadata store. + * The cache uses the same interface as the metadata store but does not support all of its methods. + * + * + */ +public class CachedMetadataStore extends InternalMetadataStoreBase implements MetadataStore { + private Logger logger = Logger.getLogger(CachedMetadataStore.class.getName()); + private static final String METADATA_STORE_ID = "ODF_METADATA_CACHE"; + private static final String STORE_PROPERTY_TYPE = "cache"; + private static final String STORE_PROPERTY_DESCRIPTION = "ODF metadata cache"; + + protected Object accessLock = new Object(); + private HashMap<String, StoredMetaDataObject> objectStore = new HashMap<String, StoredMetaDataObject>(); + private HashMap<String, ConnectionInfo> connectionInfoStore = new HashMap<String, ConnectionInfo>(); + + public CachedMetadataStore(MetaDataCache metaDataCache) { + for (StoredMetaDataObject obj : metaDataCache.getMetaDataObjects()) { + getObjects().put(obj.getMetaDataObject().getReference().getId(), obj); + logger.log(Level.FINER, "Added object with name ''{0}'' to metadata cache.", obj.getMetaDataObject().getName()); + } + for (ConnectionInfo conInfo : metaDataCache.getConnectionInfoObjects()) { + connectionInfoStore.put(conInfo.getAssetReference().getId(), conInfo); + logger.log(Level.FINER, "Added connection info object for metadata object id ''{0}'' to metadata cache.", conInfo.getAssetReference().getId()); + } + } + + protected Object getAccessLock() { + return accessLock; + } + + public static MetaDataCache retrieveMetaDataCache(MetadataStore mds, MetaDataObject metaDataObject) { + MetaDataCache cache = new MetaDataCache(); + populateMetaDataCache(cache, mds, metaDataObject); + return cache; + } + /** + * Internal methods that recursively populates the metadata store with all child objects of a given metadata object. + * If there is a @see ConnectionInfo object available for a cached metadata object + * it will be added to the cache as well. + * + * @param metaDataCache Metadata cache to be populated + * @param mds Metadata store to retrieve the cached objects from + * @param metaDataObject Given metadata object + */ + private static void populateMetaDataCache(MetaDataCache metaDataCache, MetadataStore mds, MetaDataObject metaDataObject) { + // Add current object + StoredMetaDataObject currentObject = new StoredMetaDataObject(metaDataObject); + for (String referenceType : mds.getReferenceTypes()) { + currentObject.getReferenceMap().put(referenceType, InternalMetaDataUtils.getReferenceList(mds.getReferences(referenceType, metaDataObject))); + } + metaDataCache.getMetaDataObjects().add(currentObject); + ConnectionInfo connectionInfo = mds.getConnectionInfo(metaDataObject); + + // Connection info must be cached as well because it cannot be retrieved dynamically as required parent objects might be missing from cache + if (connectionInfo != null) { + metaDataCache.getConnectionInfoObjects().add(connectionInfo); + } + + // Add child objects + for (MetaDataObject child : mds.getChildren(metaDataObject)) { + populateMetaDataCache(metaDataCache, mds, child); + } + } + + protected HashMap<String, StoredMetaDataObject> getObjects() { + return objectStore; + } + + @Override + public Properties getProperties() { + Properties props = new Properties(); + props.put(MetadataStore.STORE_PROPERTY_DESCRIPTION, STORE_PROPERTY_DESCRIPTION); + props.put(MetadataStore.STORE_PROPERTY_TYPE, STORE_PROPERTY_TYPE); + props.put(STORE_PROPERTY_ID, METADATA_STORE_ID); + return props; + } + + @Override + public void resetAllData() { + throw new UnsupportedOperationException("Method not available in this implementation of the Metadata store."); + } + + @Override + public String getRepositoryId() { + return METADATA_STORE_ID; + } + + @Override + public ConnectionInfo getConnectionInfo(MetaDataObject metaDataObject) { + return connectionInfoStore.get(metaDataObject.getReference().getId()); + } + + @Override + public List<MetaDataObjectReference> search(String query) { + throw new UnsupportedOperationException("Method not available in this implementation of the Metadata store."); + } + + @Override + public void createSampleData() { + throw new UnsupportedOperationException("Method not available in this implementation of the Metadata store."); + } + + @Override + public AnnotationPropagator getAnnotationPropagator() { + throw new UnsupportedOperationException("Method not available in this implementation of the Metadata store."); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ClassificationAnnotation.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ClassificationAnnotation.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ClassificationAnnotation.java new file mode 100755 index 0000000..8db6ec1 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ClassificationAnnotation.java @@ -0,0 +1,38 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +import java.util.List; + +import org.apache.atlas.odf.api.metadata.MetaDataObjectReference; + +public class ClassificationAnnotation extends Annotation { + + private MetaDataObjectReference classifiedObject; + private List<MetaDataObjectReference> classifyingObjects; + + public MetaDataObjectReference getClassifiedObject() { + return classifiedObject; + } + public void setClassifiedObject(MetaDataObjectReference classifiedObject) { + this.classifiedObject = classifiedObject; + } + + public List<MetaDataObjectReference> getClassifyingObjects() { + return classifyingObjects; + } + public void setClassifyingObjects(List<MetaDataObjectReference> classifyingObjects) { + this.classifyingObjects = classifyingObjects; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Column.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Column.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Column.java new file mode 100755 index 0000000..0ae4370 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Column.java @@ -0,0 +1,32 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +/** + * This class represents metadata of a column in a table + * + */ +public class Column extends MetaDataObject { + + private String dataType; + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Connection.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Connection.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Connection.java new file mode 100755 index 0000000..9a42fc2 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Connection.java @@ -0,0 +1,18 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +public abstract class Connection extends MetaDataObject { + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ConnectionInfo.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ConnectionInfo.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ConnectionInfo.java new file mode 100755 index 0000000..4884105 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ConnectionInfo.java @@ -0,0 +1,64 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +import java.util.List; + +import org.apache.atlas.odf.api.metadata.MetaDataObjectReference; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * + * General connecting info that must be extended for individual data sources + * + */ +@ApiModel(description="Object containing the information required in order to access the data behind a specific metadata object.") +public abstract class ConnectionInfo { + + @ApiModelProperty(value="Available connections for accessing the data behind the metadata object", readOnly=true, required=true) + private List<Connection> connections; + + @ApiModelProperty(value="Reference to the actual metadata object", readOnly=true, required=true) + private MetaDataObjectReference assetReference; + + @ApiModelProperty(value="Java class represeting the connection info object", hidden=true) + private String javaClass = this.getClass().getName(); // don't use JsonTypeInfo + + public List<Connection> getConnections() { + return this.connections; + } + + public void setConnections(List<Connection> connections) { + this.connections = connections; + } + + public MetaDataObjectReference getAssetReference() { + return this.assetReference; + } + + public void setAssetReference(MetaDataObjectReference assetReference) { + this.assetReference = assetReference; + } + + public String getJavaClass() { + return javaClass; + } + + public void setJavaClass(String javaClass) { + this.javaClass = javaClass; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFile.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFile.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFile.java new file mode 100755 index 0000000..2a1ad7f --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFile.java @@ -0,0 +1,39 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +/** + * This class is a metadataobject for a CSV file located at a specific URL + * + */ +public class DataFile extends RelationalDataSet { + private String encoding = "UTF-8"; + private String urlString; + + public String getUrlString() { + return urlString; + } + + public void setUrlString(String url) { + this.urlString = url; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFileFolder.java ---------------------------------------------------------------------- diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFileFolder.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFileFolder.java new file mode 100755 index 0000000..5e2a132 --- /dev/null +++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataFileFolder.java @@ -0,0 +1,18 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.odf.api.metadata.models; + +public class DataFileFolder extends MetaDataObject { + +}
