From 216411aa3ed70681c308d59875c2e104f01d95b6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 29 Aug 2016 23:05:13 -0400 Subject: [PATCH 1/2] TST: Enable Google BigQuery (pandas.io.gbq) integration testing #11089 --- .travis.yml | 6 + ci/requirements-2.7.pip | 1 + ci/travis_gbq.json.enc | Bin 0 -> 2352 bytes pandas/io/tests/test_gbq.py | 289 +++++++++++++++++++++++------------- 4 files changed, 196 insertions(+), 100 deletions(-) create mode 100644 ci/travis_gbq.json.enc diff --git a/.travis.yml b/.travis.yml index 2716fa7628d61..7acad23bdbecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -229,6 +229,12 @@ matrix: - USE_CACHE=true before_install: + - if [ -n "$encrypted_1d9d7b1f171b_iv" ]; then + openssl aes-256-cbc -K $encrypted_1d9d7b1f171b_key + -iv $encrypted_1d9d7b1f171b_iv -in ci/travis_gbq.json.enc + -out ci/travis_gbq.json -d; + export VALID_GBQ_CREDENTIALS=True; + fi - echo "before_install" - echo $VIRTUAL_ENV - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/ci/requirements-2.7.pip b/ci/requirements-2.7.pip index cc3462dbf9ed0..d16b932c8be4f 100644 --- a/ci/requirements-2.7.pip +++ b/ci/requirements-2.7.pip @@ -6,3 +6,4 @@ oauth2client==1.5.0 pathlib backports.lzma py +PyCrypto diff --git a/ci/travis_gbq.json.enc b/ci/travis_gbq.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..c2a33bbd6f26383bd7e8a7a504e626284efb5fd0 GIT binary patch literal 2352 zcmV-03D5QoiY_vZjh&7QCFrhKcFBG@`zj6HxkUamBtL*$SOfIYLQAnP$$?HCW-UzE zqY3S}bS_tytBr;XZgqTWlqlC0A?TtDDzJS4<-4yF+82AKZYaOSzyy z)LIN&*Phn|s>u2rH)V_1hyj-xu@)mBOg%_tj5_Sz6kyK>B5Gj0bp;~khYB=Ul|&X? zUFSM`<{}P#4_#PMfT#y?P!&Q=azAz#tG@DOU=aLF%RTb9pTg+mwrTZ+`_vBO5^xdb zCk{k&n*k1|x?M-4M;q$_?J$Z=GMNDL*;ETHrT|OpFalF9aJ;1NN8;rz^YfzF2c#MtNZvI;NuIJQ-M<=GHh=X9{ian$nm(H@?nOf1bgG`&RpLSr<5g9xf z2teKs?kATag6a+LsF}ejFjmcfSCRZKh(1~}uiJ(Qc@Q;)ValsMLtF!2X$O%Cb z2KMdb?&ns7GPy+RSdg<1=+QLqzgq74x1J+)2!4_{d|gtTVv9I=qfT>YNLb!NjSeg= zF|Qh88XA3rHR)>wth;QO_M(&hfA8)$QEpGgANx7DK|J`dW)T_`Xz_E!NK^R8RZg$y zc5}UIuDBt}n1#0!5GPf8Jbgag71LqHsVxL^@1qNIX|Dy=0vXV0(4^j2t$?ktEZdd5 zu_ckdLNK1WUPlJaR4^MLsqCIlhr=wrO2O}*qt8Z*MskXFh93(O!7RnBrwEDnT<`it5D0Mb#*2bx#aqC@LEJC=x_>Rx<|ygktaBRpWD z4#{MIj?XI%F|f1Z!qi;RP!vt6Ble@nmfAd}TzlXws1BJ)f5{5gri+aezIomN6ImrH zx}$i#tM@W$hzh(j)Gt+D=6S|?h}()_-~|h%S3)QyM`7f{Yf{v>p$dbYb8XdaAwacm zYIgF03~bBRJ?Q|Rm{AoSq^LSBkDa|`3tNoi02mXu+-Du+k_EUwoHMFk922)^pS;_D6#vtq~4S z0+*&E9tblkhvce%@L*}odrsPg ze1D(imA!lhnI7E+EDFG9720>Y4#l_d;0oNsr)BvjIN8`WGnc1$a?%?ycY8#Jhm$-C3s{t9ZH!5Tdr>`t41 zT)!t07R`S+w73>s@5X;v4d{Zrz<~%E?>$ry4A?zF{TOsf3y|_$p=_p^7 zyHtMEaO`#lEy8g>>v{%h!1*z-W`(rGI}x7M3P7v}4?u6$pF9q$Z>h4+;M|XMMXn-` zt;L)h+N2X->u!;3$*+|@qIVFK-FHTOWzOKyOMLi?7uHQUumZzC>x@c?*cS{IeR9pz z%j|yMgIP(6EQpB4%%ANMRmAGv^MZ8l-{UC8Un6k3C~MltE7?VC^N!9xT725P)|Gtf z&Y(8ua0ZUJO(-Sc>1rq^R0ra;Wa5&>w$UCFV36KRm<$T^2(h&JMd-wYacGQvViWbN z;Sj}nB6rj56!|*PGf00&z+`c`4W3nX4V>s9=aCW8AGAn)EiROzk#ku76;QET`eHgm z(nw)$QzY5E$?_QwzB-{3OpF_c;7(A1@_v7pYaO5JgoY(y&*&O#VUKi8dkA)N#1BEo z^s5wOm{@=f>c|t#|7>EeQqHh!uRXjICpE`%G!Z+Zt<^J-#-9iG(VG#%Nv?sI+ zbc`m4USJyzcgu?tl;%C}Ez6G@|f#&^hF+`g-yrj{hmY4yhlk+b#gV44cV?S5r%;?ge?g z#lzI?kuY1oXLg&XxdkBG8g*9plC**(x1xRs!fCuZZfAb#o*pyTq1{n<-CM+4c6lHo zqhwh;eK)Jl1X}YUP)?=oto!8X%qgNi1g>n7$x+*H3lrxcs&2-MENP(#=M;+oe_zRD zmCP_qF1Fe;UFgs(|6U79ig}b`dz4{4Eh38)&RvnO=3V=+bB@oe8weiJM6CJ5c%GQ-iz&#q=Du>_LJKa?c5%>1J4;MeQNYk^_$~ z;|WA1#Nz81yr8Jafys`4PisrSy?Jw~yQrKw#cLkq4Jq8We*d_mk#2#X^w3p=gJB>* z#!GJ%sBPy+SR&x<$od^Zj0! zidEfbN|w72WG4PR*<}{0X+HTW38KvQlnKe|LO@K*{nS!xOGu^})|VMf4R={d{^$ZY Wc%~RC+CiWM`BrrE1b(~# literal 0 HcmV?d00001 diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 4b71192c907f8..7757950592da5 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -4,6 +4,8 @@ import pytz import platform from time import sleep +import os +import logging import numpy as np @@ -21,7 +23,11 @@ PRIVATE_KEY_JSON_PATH = None PRIVATE_KEY_JSON_CONTENTS = None -DATASET_ID = 'pydata_pandas_bq_testing' +if compat.PY3: + DATASET_ID = 'pydata_pandas_bq_testing_py3' +else: + DATASET_ID = 'pydata_pandas_bq_testing_py2' + TABLE_ID = 'new_test' DESTINATION_TABLE = "{0}.{1}".format(DATASET_ID + "1", TABLE_ID) @@ -35,25 +41,50 @@ def _skip_if_no_project_id(): - if not PROJECT_ID: + if not _get_project_id(): raise nose.SkipTest( "Cannot run integration tests without a project id") def _skip_if_no_private_key_path(): - if not PRIVATE_KEY_JSON_PATH: + if not _get_private_key_path(): raise nose.SkipTest("Cannot run integration tests without a " "private key json file path") def _skip_if_no_private_key_contents(): - if not PRIVATE_KEY_JSON_CONTENTS: + if not _get_private_key_contents(): raise nose.SkipTest("Cannot run integration tests without a " "private key json contents") - _skip_if_no_project_id() - _skip_if_no_private_key_path() - _skip_if_no_private_key_contents() + +def _in_travis_environment(): + return 'TRAVIS_BUILD_DIR' in os.environ and \ + 'VALID_GBQ_CREDENTIALS' in os.environ + + +def _get_project_id(): + if _in_travis_environment(): + return 'pandas-travis' + else: + return PROJECT_ID + + +def _get_private_key_path(): + if _in_travis_environment(): + return os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', + 'travis_gbq.json']) + else: + return PRIVATE_KEY_JSON_PATH + + +def _get_private_key_contents(): + if _in_travis_environment(): + with open(os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', + 'travis_gbq.json'])) as f: + return f.read() + else: + return PRIVATE_KEY_JSON_CONTENTS def _test_imports(): @@ -144,18 +175,22 @@ def _test_imports(): "service account support") -def test_requirements(): +def _setup_common(): try: _test_imports() except (ImportError, NotImplementedError) as import_exception: raise nose.SkipTest(import_exception) + if _in_travis_environment(): + logging.getLogger('oauth2client').setLevel(logging.ERROR) + logging.getLogger('apiclient').setLevel(logging.ERROR) + def _check_if_can_get_correct_default_credentials(): # Checks if "Application Default Credentials" can be fetched # from the environment the tests are running in. # See Issue #13577 - test_requirements() + import httplib2 try: from googleapiclient.discovery import build @@ -169,19 +204,20 @@ def _check_if_can_get_correct_default_credentials(): bigquery_service = build('bigquery', 'v2', http=http) jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} - jobs.insert(projectId=PROJECT_ID, body=job_data).execute() + jobs.insert(projectId=_get_project_id(), body=job_data).execute() return True except: return False def clean_gbq_environment(private_key=None): - dataset = gbq._Dataset(PROJECT_ID, private_key=private_key) + dataset = gbq._Dataset(_get_project_id(), private_key=private_key) for i in range(1, 10): if DATASET_ID + str(i) in dataset.datasets(): dataset_id = DATASET_ID + str(i) - table = gbq._Table(PROJECT_ID, dataset_id, private_key=private_key) + table = gbq._Table(_get_project_id(), dataset_id, + private_key=private_key) for j in range(1, 20): if TABLE_ID + str(j) in dataset.tables(dataset_id): table.delete(TABLE_ID + str(j)) @@ -215,11 +251,11 @@ def test_generate_bq_schema_deprecated(): class TestGBQConnectorIntegration(tm.TestCase): def setUp(self): - test_requirements() - + _setup_common() _skip_if_no_project_id() - self.sut = gbq.GbqConnector(PROJECT_ID) + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) def test_should_be_able_to_make_a_connector(self): self.assertTrue(self.sut is not None, @@ -259,13 +295,13 @@ def test_get_application_default_credentials_returns_credentials(self): class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): def setUp(self): - test_requirements() + _setup_common() _skip_if_no_project_id() _skip_if_no_private_key_path() - self.sut = gbq.GbqConnector(PROJECT_ID, - private_key=PRIVATE_KEY_JSON_PATH) + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) def test_should_be_able_to_make_a_connector(self): self.assertTrue(self.sut is not None, @@ -290,13 +326,13 @@ def test_should_be_able_to_get_results_from_query(self): class TestGBQConnectorServiceAccountKeyContentsIntegration(tm.TestCase): def setUp(self): - test_requirements() + _setup_common() _skip_if_no_project_id() - _skip_if_no_private_key_contents() + _skip_if_no_private_key_path() - self.sut = gbq.GbqConnector(PROJECT_ID, - private_key=PRIVATE_KEY_JSON_CONTENTS) + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) def test_should_be_able_to_make_a_connector(self): self.assertTrue(self.sut is not None, @@ -321,7 +357,7 @@ def test_should_be_able_to_get_results_from_query(self): class GBQUnitTests(tm.TestCase): def setUp(self): - test_requirements() + _setup_common() def test_import_google_api_python_client(self): if compat.PY2: @@ -396,12 +432,12 @@ def test_read_gbq_with_empty_private_key_file_should_fail(self): private_key=empty_file_path) def test_read_gbq_with_corrupted_private_key_json_should_fail(self): - _skip_if_no_private_key_contents() + _skip_if_no_private_key_path() with tm.assertRaises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', - private_key=re.sub('[a-z]', '9', PRIVATE_KEY_JSON_CONTENTS)) + private_key=re.sub('[a-z]', '9', _get_private_key_path())) class TestReadGBQIntegration(tm.TestCase): @@ -414,7 +450,7 @@ def setUpClass(cls): _skip_if_no_project_id() - test_requirements() + _setup_common() def setUp(self): # - PER-TEST FIXTURES - @@ -435,87 +471,108 @@ def tearDown(self): # executed. pass + def test_should_read_as_user_account(self): + if _in_travis_environment(): + raise nose.SkipTest("Cannot run local auth in travis environment") + + query = 'SELECT "PI" as VALID_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id()) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + def test_should_read_as_service_account_with_key_path(self): _skip_if_no_private_key_path() query = 'SELECT "PI" as VALID_STRING' - df = gbq.read_gbq(query, project_id=PROJECT_ID, - private_key=PRIVATE_KEY_JSON_PATH) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) def test_should_read_as_service_account_with_key_contents(self): _skip_if_no_private_key_contents() query = 'SELECT "PI" as VALID_STRING' - df = gbq.read_gbq(query, project_id=PROJECT_ID, - private_key=PRIVATE_KEY_JSON_CONTENTS) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_contents()) tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) def test_should_properly_handle_valid_strings(self): query = 'SELECT "PI" as VALID_STRING' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) def test_should_properly_handle_empty_strings(self): query = 'SELECT "" as EMPTY_STRING' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'EMPTY_STRING': [""]})) def test_should_properly_handle_null_strings(self): query = 'SELECT STRING(NULL) as NULL_STRING' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'NULL_STRING': [None]})) def test_should_properly_handle_valid_integers(self): query = 'SELECT INTEGER(3) as VALID_INTEGER' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'VALID_INTEGER': [3]})) def test_should_properly_handle_null_integers(self): query = 'SELECT INTEGER(NULL) as NULL_INTEGER' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'NULL_INTEGER': [np.nan]})) def test_should_properly_handle_valid_floats(self): query = 'SELECT PI() as VALID_FLOAT' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame( {'VALID_FLOAT': [3.141592653589793]})) def test_should_properly_handle_null_floats(self): query = 'SELECT FLOAT(NULL) as NULL_FLOAT' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'NULL_FLOAT': [np.nan]})) def test_should_properly_handle_timestamp_unix_epoch(self): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") as UNIX_EPOCH' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame( {'UNIX_EPOCH': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) def test_should_properly_handle_arbitrary_timestamp(self): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") as VALID_TIMESTAMP' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({ 'VALID_TIMESTAMP': [np.datetime64('2004-09-15T05:00:00.000000Z')] })) def test_should_properly_handle_null_timestamp(self): query = 'SELECT TIMESTAMP(NULL) as NULL_TIMESTAMP' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'NULL_TIMESTAMP': [NaT]})) def test_should_properly_handle_true_boolean(self): query = 'SELECT BOOLEAN(TRUE) as TRUE_BOOLEAN' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'TRUE_BOOLEAN': [True]})) def test_should_properly_handle_false_boolean(self): query = 'SELECT BOOLEAN(FALSE) as FALSE_BOOLEAN' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'FALSE_BOOLEAN': [False]})) def test_should_properly_handle_null_boolean(self): query = 'SELECT BOOLEAN(NULL) as NULL_BOOLEAN' - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({'NULL_BOOLEAN': [None]})) def test_unicode_string_conversion_and_normalization(self): @@ -530,13 +587,15 @@ def test_unicode_string_conversion_and_normalization(self): query = 'SELECT "{0}" as UNICODE_STRING'.format(unicode_string) - df = gbq.read_gbq(query, project_id=PROJECT_ID) + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) tm.assert_frame_equal(df, correct_test_datatype) def test_index_column(self): query = "SELECT 'a' as STRING_1, 'b' as STRING_2" - result_frame = gbq.read_gbq( - query, project_id=PROJECT_ID, index_col="STRING_1") + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + index_col="STRING_1", + private_key=_get_private_key_path()) correct_frame = DataFrame( {'STRING_1': ['a'], 'STRING_2': ['b']}).set_index("STRING_1") tm.assert_equal(result_frame.index.name, correct_frame.index.name) @@ -544,8 +603,9 @@ def test_index_column(self): def test_column_order(self): query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" col_order = ['STRING_3', 'STRING_1', 'STRING_2'] - result_frame = gbq.read_gbq( - query, project_id=PROJECT_ID, col_order=col_order) + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + col_order=col_order, + private_key=_get_private_key_path()) correct_frame = DataFrame({'STRING_1': ['a'], 'STRING_2': [ 'b'], 'STRING_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) @@ -553,8 +613,9 @@ def test_column_order(self): def test_column_order_plus_index(self): query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" col_order = ['STRING_3', 'STRING_2'] - result_frame = gbq.read_gbq(query, project_id=PROJECT_ID, - index_col='STRING_1', col_order=col_order) + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + index_col='STRING_1', col_order=col_order, + private_key=_get_private_key_path()) correct_frame = DataFrame( {'STRING_1': ['a'], 'STRING_2': ['b'], 'STRING_3': ['c']}) correct_frame.set_index('STRING_1', inplace=True) @@ -564,16 +625,19 @@ def test_column_order_plus_index(self): def test_malformed_query(self): with tm.assertRaises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) def test_bad_project_id(self): with tm.assertRaises(gbq.GenericGBQException): - gbq.read_gbq("SELECT 1", project_id='001') + gbq.read_gbq("SELECT 1", project_id='001', + private_key=_get_private_key_path()) def test_bad_table_name(self): with tm.assertRaises(gbq.GenericGBQException): gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) def test_download_dataset_larger_than_200k_rows(self): test_size = 200005 @@ -582,7 +646,8 @@ def test_download_dataset_larger_than_200k_rows(self): df = gbq.read_gbq("SELECT id FROM [publicdata:samples.wikipedia] " "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" .format(test_size), - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) self.assertEqual(len(df.drop_duplicates()), test_size) def test_zero_rows(self): @@ -590,7 +655,8 @@ def test_zero_rows(self): df = gbq.read_gbq("SELECT title, id " "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) page_array = np.zeros( (0,), dtype=[('title', object), ('id', np.dtype(float))]) expected_result = DataFrame(page_array, columns=['title', 'id']) @@ -602,13 +668,15 @@ def test_legacy_sql(self): # Test that a legacy sql statement fails when # setting dialect='standard' with tm.assertRaises(gbq.GenericGBQException): - gbq.read_gbq(legacy_sql, project_id=PROJECT_ID, - dialect='standard') + gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + dialect='standard', + private_key=_get_private_key_path()) # Test that a legacy sql statement succeeds when # setting dialect='legacy' - df = gbq.read_gbq(legacy_sql, project_id=PROJECT_ID, - dialect='legacy') + df = gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + dialect='legacy', + private_key=_get_private_key_path()) self.assertEqual(len(df.drop_duplicates()), 10) def test_standard_sql(self): @@ -618,12 +686,14 @@ def test_standard_sql(self): # Test that a standard sql statement fails when using # the legacy SQL dialect (default value) with tm.assertRaises(gbq.GenericGBQException): - gbq.read_gbq(standard_sql, project_id=PROJECT_ID) + gbq.read_gbq(standard_sql, project_id=_get_project_id(), + private_key=_get_private_key_path()) # Test that a standard sql statement succeeds when # setting dialect='standard' - df = gbq.read_gbq(standard_sql, project_id=PROJECT_ID, - dialect='standard') + df = gbq.read_gbq(standard_sql, project_id=_get_project_id(), + dialect='standard', + private_key=_get_private_key_path()) self.assertEqual(len(df.drop_duplicates()), 10) def test_invalid_option_for_sql_dialect(self): @@ -632,13 +702,14 @@ def test_invalid_option_for_sql_dialect(self): # Test that an invalid option for `dialect` raises ValueError with tm.assertRaises(ValueError): - gbq.read_gbq(sql_statement, project_id=PROJECT_ID, - dialect='invalid') + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + dialect='invalid', + private_key=_get_private_key_path()) # Test that a correct option for dialect succeeds # to make sure ValueError was due to invalid dialect - gbq.read_gbq(sql_statement, project_id=PROJECT_ID, - dialect='standard') + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + dialect='standard', private_key=_get_private_key_path()) class TestToGBQIntegration(tm.TestCase): @@ -656,18 +727,22 @@ def setUpClass(cls): _skip_if_no_project_id() - test_requirements() - clean_gbq_environment() + _setup_common() + clean_gbq_environment(_get_private_key_path()) - gbq._Dataset(PROJECT_ID).create(DATASET_ID + "1") + gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path() + ).create(DATASET_ID + "1") def setUp(self): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - self.dataset = gbq._Dataset(PROJECT_ID) - self.table = gbq._Table(PROJECT_ID, DATASET_ID + "1") + self.dataset = gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path()) + self.table = gbq._Table(_get_project_id(), DATASET_ID + "1", + private_key=_get_private_key_path()) @classmethod def tearDownClass(cls): @@ -675,7 +750,7 @@ def tearDownClass(cls): # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - clean_gbq_environment() + clean_gbq_environment(_get_private_key_path()) def tearDown(self): # - PER-TEST FIXTURES - @@ -689,13 +764,15 @@ def test_upload_data(self): test_size = 20001 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, PROJECT_ID, chunksize=10000) + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" .format(destination_table), - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], test_size) def test_upload_data_if_table_exists_fail(self): @@ -707,11 +784,13 @@ def test_upload_data_if_table_exists_fail(self): # Test the default value of if_exists is 'fail' with tm.assertRaises(gbq.TableCreationError): - gbq.to_gbq(df, destination_table, PROJECT_ID) + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=_get_private_key_path()) # Test the if_exists parameter with value 'fail' with tm.assertRaises(gbq.TableCreationError): - gbq.to_gbq(df, destination_table, PROJECT_ID, if_exists='fail') + gbq.to_gbq(df, destination_table, _get_project_id(), + if_exists='fail', private_key=_get_private_key_path()) def test_upload_data_if_table_exists_append(self): destination_table = DESTINATION_TABLE + "3" @@ -721,22 +800,26 @@ def test_upload_data_if_table_exists_append(self): df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, destination_table, PROJECT_ID, chunksize=10000) + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) # Test the if_exists parameter with value 'append' - gbq.to_gbq(df, destination_table, PROJECT_ID, if_exists='append') + gbq.to_gbq(df, destination_table, _get_project_id(), + if_exists='append', private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" .format(destination_table), - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], test_size * 2) # Try inserting with a different schema, confirm failure with tm.assertRaises(gbq.InvalidSchema): gbq.to_gbq(df_different_schema, destination_table, - PROJECT_ID, if_exists='append') + _get_project_id(), if_exists='append', + private_key=_get_private_key_path()) def test_upload_data_if_table_exists_replace(self): destination_table = DESTINATION_TABLE + "4" @@ -746,17 +829,20 @@ def test_upload_data_if_table_exists_replace(self): df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, destination_table, PROJECT_ID, chunksize=10000) + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) # Test the if_exists parameter with the value 'replace'. gbq.to_gbq(df_different_schema, destination_table, - PROJECT_ID, if_exists='replace') + _get_project_id(), if_exists='replace', + private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" .format(destination_table), - project_id=PROJECT_ID) + project_id=_get_project_id(), + private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], 5) def test_google_upload_errors_should_raise_exception(self): @@ -769,7 +855,8 @@ def test_google_upload_errors_should_raise_exception(self): index=range(2)) with tm.assertRaises(gbq.StreamingInsertError): - gbq.to_gbq(bad_df, destination_table, PROJECT_ID, verbose=True) + gbq.to_gbq(bad_df, destination_table, _get_project_id(), + verbose=True, private_key=_get_private_key_path()) def test_generate_schema(self): df = tm.makeMixedDataFrame() @@ -828,7 +915,9 @@ def test_list_dataset(self): def test_list_table_zero_results(self): dataset_id = DATASET_ID + "2" self.dataset.create(dataset_id) - table_list = gbq._Dataset(PROJECT_ID).tables(dataset_id) + table_list = gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path() + ).tables(dataset_id) self.assertEqual(len(table_list), 0, 'Expected gbq.list_table() to return 0') @@ -854,7 +943,7 @@ def test_dataset_exists(self): def create_table_data_dataset_does_not_exist(self): dataset_id = DATASET_ID + "6" table_id = TABLE_ID + "1" - table_with_new_dataset = gbq._Table(PROJECT_ID, dataset_id) + table_with_new_dataset = gbq._Table(_get_project_id(), dataset_id) df = make_mixed_dataframe_v2(10) table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) self.assertTrue(self.dataset.exists(dataset_id), @@ -884,8 +973,8 @@ def setUpClass(cls): _skip_if_no_project_id() _skip_if_no_private_key_path() - test_requirements() - clean_gbq_environment(PRIVATE_KEY_JSON_PATH) + _setup_common() + clean_gbq_environment(_get_private_key_path()) def setUp(self): # - PER-TEST FIXTURES - @@ -899,7 +988,7 @@ def tearDownClass(cls): # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - clean_gbq_environment(PRIVATE_KEY_JSON_PATH) + clean_gbq_environment(_get_private_key_path()) def tearDown(self): # - PER-TEST FIXTURES - @@ -913,15 +1002,15 @@ def test_upload_data_as_service_account_with_key_path(self): test_size = 10 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, PROJECT_ID, chunksize=10000, - private_key=PRIVATE_KEY_JSON_PATH) + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq( "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), - project_id=PROJECT_ID, - private_key=PRIVATE_KEY_JSON_PATH) + project_id=_get_project_id(), + private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], test_size) @@ -940,11 +1029,11 @@ def setUpClass(cls): # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. + _setup_common() _skip_if_no_project_id() _skip_if_no_private_key_contents() - test_requirements() - clean_gbq_environment(PRIVATE_KEY_JSON_CONTENTS) + clean_gbq_environment(_get_private_key_contents()) def setUp(self): # - PER-TEST FIXTURES - @@ -958,7 +1047,7 @@ def tearDownClass(cls): # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - clean_gbq_environment(PRIVATE_KEY_JSON_CONTENTS) + clean_gbq_environment(_get_private_key_contents()) def tearDown(self): # - PER-TEST FIXTURES - @@ -972,15 +1061,15 @@ def test_upload_data_as_service_account_with_key_contents(self): test_size = 10 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, PROJECT_ID, chunksize=10000, - private_key=PRIVATE_KEY_JSON_CONTENTS) + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_contents()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq( "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), - project_id=PROJECT_ID, - private_key=PRIVATE_KEY_JSON_CONTENTS) + project_id=_get_project_id(), + private_key=_get_private_key_contents()) self.assertEqual(result['NUM_ROWS'][0], test_size) if __name__ == '__main__': From 1a13487e594625f76041da5fe418e15b052a7709 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 29 Aug 2016 23:06:20 -0400 Subject: [PATCH 2/2] Update encrypted credentials file for testing --- .travis.yml | 6 +++--- ci/travis_gbq.json.enc | Bin 2352 -> 2352 bytes 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7acad23bdbecc..bd96609985c03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -229,9 +229,9 @@ matrix: - USE_CACHE=true before_install: - - if [ -n "$encrypted_1d9d7b1f171b_iv" ]; then - openssl aes-256-cbc -K $encrypted_1d9d7b1f171b_key - -iv $encrypted_1d9d7b1f171b_iv -in ci/travis_gbq.json.enc + - if [ -n "$encrypted_4a66b2b60580_iv" ]; then + openssl aes-256-cbc -K $encrypted_4a66b2b60580_key + -iv $encrypted_4a66b2b60580_iv -in ci/travis_gbq.json.enc -out ci/travis_gbq.json -d; export VALID_GBQ_CREDENTIALS=True; fi diff --git a/ci/travis_gbq.json.enc b/ci/travis_gbq.json.enc index c2a33bbd6f26383bd7e8a7a504e626284efb5fd0..c7a275117578f72d6c00e782b332a0c965813a99 100644 GIT binary patch literal 2352 zcmV-03D5S>z9SwEsW^8OwHy?Pr|P+=bZ}@jA!9dBM@x3T{6Qh32Z}B_+IbYviGAu8naVn4 zI)hfL`U=R2pu%>&v?z}?78B$jH-Yp1 z9-lSwyTdGHfK5qK^=phiKIVY?*>A2~Wpz53*3aqxu+fDnZ?x*~x2+e#Qy5@dG?6Rx z+w{&PjBGq9v5nJYUOB}ImbD(?bkSz_ejB97+1*-=DNR41VU_F@3`~$s!mzTPJHpkf zHQ2ymHt=dS#D5PIr5E}sax(X@RUrAaH>DH0M2GVNDKPjyhdPam{T~ZLbK7p*C|3#k zPgAKa*Ik-rnW#|9)tXq3qb58c3`;`}D^HnaqzNI3JX=PON@d}U^)r%vbFI9_M|ogE%wVN>M1=t$o_x?LC#{pvPjaS%NHj)Pw6c$Z)~rabOPr-@se9dJJt%GDq6^8IhA z8FiO<;G0D+_Bvhz#+j5@A<9@M{E^}9I|__a5WW#MTmDdk=x{|{aqr_}Er}D&zu+rR z=M?`oX3~yxX?n<1&pH*wi6#d58)fQ5<&OoN$4_@!RWZX)D+|2?Whr{*R4?(Qo*ZRH zJ&Zcb@Gyct=2&Y&R|E1K*1e!Ur65xZFbM}&8BrhNyYK_mx4~(9i|byX7;Ok&3ZWOa zFG2<;gxH6$s?PQsEw*)*VuSfR>kv0Qr#K=RlxLjai?AGuC!uNaU^U1Q@rDYw(%;TU zH^-2K!HF1Kistql`b*~it|Z-9Ha9A(KslOfuHNGURcYVA*vQh&W2n6JA+RE$r1Nge zO-(EdGDr)hZKGCibO=Rb=AUrYDRo;!mAIH$S_7@-uXAr8hMQ{*+GWf=qS=p23=XEJ ziGz&Z6$9Q_BoPShChn|5>I8VFZnA{G(Dg{1&bAE`SuG31Zl`JA_6z|mvPl!oz%hVN za8{xkOL_qT^$o_`t7ixaSzBYc92IXB3i%3X!Xhf++PY)e~bwx!t^Hfp$Gx?^4-!%G}`&P zYh%p-Pu}13xAB|fD5|vvM+r_1PXJBbBEs3$Lo4>CyVtj`V{2 z-sd`*I~K!M%|w@(j7wBV0^&iYs62y!^=04BjrKBekM2#o@)m|T{i;;^aae|Q5a0Vd z48biB4SxFo58eY3{VY-5P^%sEyfthbbX;LqN~gK8edQk^=|J;LBqp9|_VXo5nz(h& z-<$BZAy_`Pr>Fj@hh7Bot-05z7C3vw(GD3;)tBpF5WPlJzTT{Zul5p`ZuL&(5pZk` z#Ngd|sibi3(q_5Q&>W_Ot%C z9X^ge(FuZ3mX$sd4RnEv`7Mt!=%h)6Y0j=OtddmG3MdvP0D}bZT=(M%mq*}oU5Q$#Avnzsm_bn#JhkctIbYdLZ)7< zIjj9ronx#iUThzvfY)`vA6zQa}ZpbPSdvK3e!!mEf*`>4Ks+AIWX?t z-wbq}`eM^;1RH}keQ_R?;jnVfwG)Hnb`JaV4{$KhspnoAfc=U<%ofQ4pR_}Q<}M&O>X&$NEe=Yosx>^9 z{5`&e2MmIjqX`cL~0)B1Oet-n`|?KGo_ti{paTwsh>P$TH97+YQt4tyAL>;i$=7P#1VEY8dJV^R0;GZ3AD6v7`j zb{zgK5TVDoDc^I<6R^Y*EZlS+jemjgJfS;X3d};5`O1Nd_`FQ5h(!lV5ASz?P$Ov?Ki zovBtzxr_&?YJ5YQ|06~2-A>uD56c+aetA6F{MSUK4<-4yF+82AKZYaOSzyy z)LIN&*Phn|s>u2rH)V_1hyj-xu@)mBOg%_tj5_Sz6kyK>B5Gj0bp;~khYB=Ul|&X? zUFSM`<{}P#4_#PMfT#y?P!&Q=azAz#tG@DOU=aLF%RTb9pTg+mwrTZ+`_vBO5^xdb zCk{k&n*k1|x?M-4M;q$_?J$Z=GMNDL*;ETHrT|OpFalF9aJ;1NN8;rz^YfzF2c#MtNZvI;NuIJQ-M<=GHh=X9{ian$nm(H@?nOf1bgG`&RpLSr<5g9xf z2teKs?kATag6a+LsF}ejFjmcfSCRZKh(1~}uiJ(Qc@Q;)ValsMLtF!2X$O%Cb z2KMdb?&ns7GPy+RSdg<1=+QLqzgq74x1J+)2!4_{d|gtTVv9I=qfT>YNLb!NjSeg= zF|Qh88XA3rHR)>wth;QO_M(&hfA8)$QEpGgANx7DK|J`dW)T_`Xz_E!NK^R8RZg$y zc5}UIuDBt}n1#0!5GPf8Jbgag71LqHsVxL^@1qNIX|Dy=0vXV0(4^j2t$?ktEZdd5 zu_ckdLNK1WUPlJaR4^MLsqCIlhr=wrO2O}*qt8Z*MskXFh93(O!7RnBrwEDnT<`it5D0Mb#*2bx#aqC@LEJC=x_>Rx<|ygktaBRpWD z4#{MIj?XI%F|f1Z!qi;RP!vt6Ble@nmfAd}TzlXws1BJ)f5{5gri+aezIomN6ImrH zx}$i#tM@W$hzh(j)Gt+D=6S|?h}()_-~|h%S3)QyM`7f{Yf{v>p$dbYb8XdaAwacm zYIgF03~bBRJ?Q|Rm{AoSq^LSBkDa|`3tNoi02mXu+-Du+k_EUwoHMFk922)^pS;_D6#vtq~4S z0+*&E9tblkhvce%@L*}odrsPg ze1D(imA!lhnI7E+EDFG9720>Y4#l_d;0oNsr)BvjIN8`WGnc1$a?%?ycY8#Jhm$-C3s{t9ZH!5Tdr>`t41 zT)!t07R`S+w73>s@5X;v4d{Zrz<~%E?>$ry4A?zF{TOsf3y|_$p=_p^7 zyHtMEaO`#lEy8g>>v{%h!1*z-W`(rGI}x7M3P7v}4?u6$pF9q$Z>h4+;M|XMMXn-` zt;L)h+N2X->u!;3$*+|@qIVFK-FHTOWzOKyOMLi?7uHQUumZzC>x@c?*cS{IeR9pz z%j|yMgIP(6EQpB4%%ANMRmAGv^MZ8l-{UC8Un6k3C~MltE7?VC^N!9xT725P)|Gtf z&Y(8ua0ZUJO(-Sc>1rq^R0ra;Wa5&>w$UCFV36KRm<$T^2(h&JMd-wYacGQvViWbN z;Sj}nB6rj56!|*PGf00&z+`c`4W3nX4V>s9=aCW8AGAn)EiROzk#ku76;QET`eHgm z(nw)$QzY5E$?_QwzB-{3OpF_c;7(A1@_v7pYaO5JgoY(y&*&O#VUKi8dkA)N#1BEo z^s5wOm{@=f>c|t#|7>EeQqHh!uRXjICpE`%G!Z+Zt<^J-#-9iG(VG#%Nv?sI+ zbc`m4USJyzcgu?tl;%C}Ez6G@|f#&^hF+`g-yrj{hmY4yhlk+b#gV44cV?S5r%;?ge?g z#lzI?kuY1oXLg&XxdkBG8g*9plC**(x1xRs!fCuZZfAb#o*pyTq1{n<-CM+4c6lHo zqhwh;eK)Jl1X}YUP)?=oto!8X%qgNi1g>n7$x+*H3lrxcs&2-MENP(#=M;+oe_zRD zmCP_qF1Fe;UFgs(|6U79ig}b`dz4{4Eh38)&RvnO=3V=+bB@oe8weiJM6CJ5c%GQ-iz&#q=Du>_LJKa?c5%>1J4;MeQNYk^_$~ z;|WA1#Nz81yr8Jafys`4PisrSy?Jw~yQrKw#cLkq4Jq8We*d_mk#2#X^w3p=gJB>* z#!GJ%sBPy+SR&x<$od^Zj0! zidEfbN|w72WG4PR*<}{0X+HTW38KvQlnKe|LO@K*{nS!xOGu^})|VMf4R={d{^$ZY Wc%~RC+CiWM`BrrE1b(~#