Repository: beam Updated Branches: refs/heads/master 9c5454287 -> 0af972095
http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf new file mode 100644 index 0000000..c42d71c --- /dev/null +++ b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf @@ -0,0 +1,42 @@ +##fileformat=VCFv4.2 +##fileDate=20090805 +##source=myImputationProgramV3.1 +##phasing=partial +##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data"> +##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth"> +##INFO=<ID=AF,Number=A,Type=Float,Description="Allele Frequency"> +##INFO=<ID=AA,Number=1,Type=String,Description="Ancestral Allele"> +##INFO=<ID=DB,Number=0,Type=Flag,Description="dbSNP membership, build 129"> +##INFO=<ID=H2,Number=0,Type=Flag,Description="HapMap2 membership"> +##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant (with unïcodé)"> +##INFO=<ID=END,Number=1,Type=Integer,Description="End position of variant"> +##FILTER=<ID=q10,Description="Quality below 10"> +##FILTER=<ID=s50,Description="Less than 50% of samples have data (with \\ backslash)"> +##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype"> +##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality"> +##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Read Depth"> +##FORMAT=<ID=HQ,Number=2,Type=Integer,Description="Haplotype Quality"> +##FORMAT=<ID=GL,Number=G,Type=Integer,Description="Genotype Likelihood"> +##reference=file:/lustre/scratch105/projects/g1k/ref/main_project/human_g1k_v37.fasta +##contig=<ID=19,length=59128983,md5=1aacd71f30db8e561810913e0b72636d,species="Homo Sapiens"> +##contig=<ID=20,length=63025520,md5=0dec9660ec1efaaf33281c0d5ea2560f,species="Homo Sapiens"> +##contig=<ID=Y,length=63025520,md5=0dec9660ec1efaaf33281c0d5ea2560f,species="Homo Sapiens"> +##SAMPLE=<ID=Blood,Genomes=Germline,Mixture=1.,Description="Patient germline genome"> +##SAMPLE=<ID=TissueSample,Genomes=Germline;Tumor,Mixture=.3;.7,Description="Patient germline genome;Patient tumor genome"> +##PEDIGREE=<Derived=ID2,Original=ID1> +##PEDIGREE=<Child=CHILD-GENOME-ID,Mother=MOTHER-GENOME-ID,Father=FATHER-GENOME-ID> +##pedigreeDB=url +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA00001 NA00002 NA00003 +19 14370 rs6054257 G A 29 PASS NS=3;DP=14;AF=0.5;DB;H2 GT:GQ:DP:HQ 0|0:48:1:51,51 1|0:48:8:51,51 1/1:43:5:.,. +20 17330 . T A 3 q10 NS=3;DP=11;AF=0.017 GT:GQ:DP:HQ 0|0:49:3:58,50 0|1:3:5:65,3 0/0:41:3 +20 1110696 rs6040355 A G,T 67 PASS NS=2;DP=10;AF=0.333,0.667;AA=T;DB GT:GQ:DP:HQ 1|2:21:6:23,27 2|1:2:0:18,2 2/2:35:4 +20 1230237 . T . 47 PASS NS=3;DP=13;AA=T GT:GQ:DP:HQ 0|0:54:7:56,60 0|0:48:4:51,51 0/0:61:2 +20 1234567 microsat1 GTC G,GTCTC 50 PASS NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/2:17:2 1/1:40:3 +20 2234567 . C [13:123457[ACGC 50 PASS SVTYPE=BÃD;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3 +20 2234568 . C .TC 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3 +20 2234569 . C CT. 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3 +20 3234569 . C <SYMBOLIC> 50 PASS END=3235677;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3 +20 4234569 . N .[13:123457[ 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 ./.:40:3 +20 5234569 . N [13:123457[. 50 PASS SVTYPE=BND;NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/1:17:2 1/1:40:3 +Y 17330 . T A 3 q10 NS=3;DP=11;AF=0.017 GT:GL 0:0,49 0:0,3 1:41,0 +HLA-A*01:01:01:01 1 . N T 50 PASS END=1;NS=3;DP=9;AA=G GT:GQ:DP:HQ 0|0:48:1:51,51 1|0:48:8:51,51 1/1:43:5:.,. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz new file mode 100644 index 0000000..4208e3e Binary files /dev/null and b/sdks/python/apache_beam/testing/data/vcf/valid-4.2.vcf.gz differ http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/apache_beam/testing/test_utils.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index 41a02cf..c28b692 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -22,6 +22,9 @@ For internal use only; no backwards-compatibility guarantees. import hashlib import imp +import os +import shutil +import tempfile from mock import Mock from mock import patch @@ -32,6 +35,43 @@ from apache_beam.utils import retry DEFAULT_HASHING_ALG = 'sha1' +class TempDir(object): + """Context Manager to create and clean-up a temporary directory.""" + + def __init__(self): + self._tempdir = tempfile.mkdtemp() + + def __enter__(self): + return self + + def __exit__(self, *args): + if os.path.exists(self._tempdir): + shutil.rmtree(self._tempdir) + + def get_path(self): + """Returns the path to the temporary directory.""" + return self._tempdir + + def create_temp_file(self, suffix='', lines=None): + """Creates a temporary file in the temporary directory. + + Args: + suffix (str): The filename suffix of the temporary file (e.g. '.txt') + lines (List[str]): A list of lines that will be written to the temporary + file. + Returns: + The name of the temporary file created. + """ + f = tempfile.NamedTemporaryFile(delete=False, + dir=self._tempdir, + suffix=suffix) + if lines: + for line in lines: + f.write(line) + + return f.name + + def compute_hash(content, hashing_alg=DEFAULT_HASHING_ALG): """Compute a hash value from a list of string.""" content.sort() http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/apache_beam/testing/test_utils_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/testing/test_utils_test.py b/sdks/python/apache_beam/testing/test_utils_test.py index 8113310..877ee39 100644 --- a/sdks/python/apache_beam/testing/test_utils_test.py +++ b/sdks/python/apache_beam/testing/test_utils_test.py @@ -56,6 +56,30 @@ class TestUtilsTest(unittest.TestCase): with self.assertRaises(RuntimeError): utils.delete_files([]) + def test_temp_dir_removes_files(self): + dir_path = '' + file_path = '' + with utils.TempDir() as tempdir: + dir_path = tempdir.get_path() + file_path = tempdir.create_temp_file() + self.assertTrue(os.path.exists(dir_path)) + self.assertTrue(os.path.exists(file_path)) + + self.assertFalse(os.path.exists(dir_path)) + self.assertFalse(os.path.exists(file_path)) + + def test_temp_file_field_correct(self): + with utils.TempDir() as tempdir: + filename = tempdir.create_temp_file( + suffix='.txt', + lines=['line1\n', 'line2\n', 'line3\n']) + self.assertTrue(filename.endswith('.txt')) + + with open(filename, 'rb') as f: + self.assertEqual(f.readline(), 'line1\n') + self.assertEqual(f.readline(), 'line2\n') + self.assertEqual(f.readline(), 'line3\n') + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/generate_pydoc.sh ---------------------------------------------------------------------- diff --git a/sdks/python/generate_pydoc.sh b/sdks/python/generate_pydoc.sh index 662bd09..9ae019c 100755 --- a/sdks/python/generate_pydoc.sh +++ b/sdks/python/generate_pydoc.sh @@ -115,6 +115,9 @@ ignore_identifiers = [ 'Set', 'Tuple', + # Ignore broken built-in type references + 'tuple', + # Ignore private classes 'apache_beam.coders.coders._PickleCoderBase', 'apache_beam.coders.coders.FastCoder', @@ -143,6 +146,9 @@ ignore_identifiers = [ # Private classes which are used within the same module 'WindowedTypeConstraint', # apache_beam.typehints.typehints + + # stdlib classes without documentation + 'unittest.case.TestCase' ] # When inferring a base class it will use ':py:class'; if inferring a function http://git-wip-us.apache.org/repos/asf/beam/blob/f22da33c/sdks/python/setup.py ---------------------------------------------------------------------- diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 5852d15..46f4f8a 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -104,6 +104,7 @@ REQUIRED_PACKAGES = [ 'oauth2client>=2.0.1,<4.0.0', 'protobuf>=3.2.0,<=3.3.0', 'pyyaml>=3.12,<4.0.0', + 'pyvcf>=0.6.8,<0.7.0', # Six 1.11.0 incompatible with apitools. # TODO(BEAM-2964): Remove the upper bound. 'six>=1.9,<1.11', @@ -160,7 +161,7 @@ setuptools.setup( author_email=PACKAGE_EMAIL, packages=setuptools.find_packages(), package_data={'apache_beam': [ - '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'testing/data/*']}, + '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'testing/data/*.yaml']}, ext_modules=cythonize([ 'apache_beam/**/*.pyx', 'apache_beam/coders/coder_impl.py',
