Skip to content

Commit 4b5e825

Browse files
authored
Follow up fix for merging errors from batch BigQuery inserts (#1348). (#1350)
* Follow up fix for merging errors from batch BigQuery inserts (#1348). * fix pylint false positive
1 parent 5e32deb commit 4b5e825

File tree

1 file changed

+12
-2
lines changed

1 file changed

+12
-2
lines changed

src/python/google_cloud_utils/big_query.py

+12-2
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ def insert(self, inserts):
370370
A json explained here:
371371
https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
372372
"""
373-
result = None
373+
result = {}
374374
for i in range(0, len(inserts), INSERT_BATCH_SIZE):
375375
response = self._insert_batch(
376376
inserts[i:min(len(inserts), i + INSERT_BATCH_SIZE)])
@@ -379,8 +379,18 @@ def insert(self, inserts):
379379
time.sleep(1)
380380

381381
if not result:
382+
# Use result from the first batch, appending errors from the rest.
382383
result = response
383384
else:
384-
result['insertErrors'].extend(response['insertErrors'])
385+
# If there are new errors from the current batch, append to the result.
386+
new_errors = response.get('insertErrors')
387+
if not new_errors:
388+
continue
389+
390+
# Apparently result may not have errors, be careful.
391+
if result.get('insertErrors'):
392+
result['insertErrors'].extend(new_errors)
393+
else:
394+
result['insertErrors'] = new_errors
385395

386396
return result

0 commit comments

Comments
 (0)