# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for load_bigquery_stats."""
import datetime
import mock
import unittest
import webapp2
import webtest
from datastore import data_types
from handlers.cron import load_bigquery_stats
from tests.test_libs import helpers as test_helpers
from tests.test_libs import test_utils
@test_utils.with_cloud_emulators('datastore')
class LoadBigQueryStatsTest(unittest.TestCase):
"""Test LoadBigQueryStatsTest."""
def setUp(self):
self.app = webtest.TestApp(
webapp2.WSGIApplication([('/load-bigquery-stats',
load_bigquery_stats.Handler)]))
data_types.Fuzzer(name='fuzzer', jobs=['job']).put()
data_types.Job(name='job').put()
test_helpers.patch(self, [
'google_cloud_utils.big_query.get_api_client',
'handlers.base_handler.Handler.is_cron',
'handlers.cron.load_bigquery_stats.Handler._utc_now',
])
self.mock._utc_now.return_value = datetime.datetime(2016, 9, 8) # pylint: disable=protected-access
self.mock_bigquery = mock.MagicMock()
self.mock.get_api_client.return_value = self.mock_bigquery
def test_execute(self):
"""Tests executing of cron job."""
self.app.get('/load-bigquery-stats')
self.mock_bigquery.datasets().insert.assert_has_calls([
mock.call(
projectId='test-clusterfuzz',
body={
'datasetReference': {
'projectId': 'test-clusterfuzz',
'datasetId': 'fuzzer_stats'
}
}),
mock.call().execute()
])
self.mock_bigquery.tables().insert.assert_has_calls([
mock.call(
body={
'timePartitioning': {
'type': 'DAY'
},
'tableReference': {
'projectId': 'test-clusterfuzz',
'tableId': 'JobRun',
'datasetId': 'fuzzer_stats',
},
},
datasetId='fuzzer_stats',
projectId='test-clusterfuzz'),
mock.call().execute(),
mock.call(
body={
'timePartitioning': {
'type': 'DAY'
},
'tableReference': {
'projectId': 'test-clusterfuzz',
'tableId': 'TestcaseRun',
'datasetId': 'fuzzer_stats',
},
},
datasetId='fuzzer_stats',
projectId='test-clusterfuzz'),
mock.call().execute(),
])
self.mock_bigquery.jobs().insert.assert_has_calls(
[
mock.call(
body={
'configuration': {
'load': {
'destinationTable': {
'projectId': 'test-clusterfuzz',
'tableId': 'JobRun$20160907',
'datasetId': 'fuzzer_stats'
},
'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
'writeDisposition':
'WRITE_TRUNCATE',
'sourceUris': [
'gs://test-bigquery-bucket/fuzzer/JobRun/date/'
'20160907/*.json'
],
'sourceFormat':
'NEWLINE_DELIMITED_JSON',
'schema': {
'fields': [{
'type': 'INTEGER',
'name': 'testcases_executed',
'mode': 'NULLABLE'
}, {
'type': 'INTEGER',
'name': 'build_revision',
'mode': 'NULLABLE'
}, {
'type': 'INTEGER',
'name': 'new_crashes',
'mode': 'NULLABLE'
}, {
'type': 'STRING',
'name': 'job',
'mode': 'NULLABLE'
}, {
'type': 'FLOAT',
'name': 'timestamp',
'mode': 'NULLABLE'
}, {
'fields': [{
'type': 'STRING',
'name': 'crash_type',
'mode': 'NULLABLE'
}, {
'type': 'BOOLEAN',
'name': 'is_new',
'mode': 'NULLABLE'
}, {
'type': 'STRING',
'name': 'crash_state',
'mode': 'NULLABLE'
}, {
'type': 'BOOLEAN',
'name': 'security_flag',
'mode': 'NULLABLE'
}, {
'type': 'INTEGER',
'name': 'count',
'mode': 'NULLABLE'
}],
'type':
'RECORD',
'name':
'crashes',
'mode':
'REPEATED'
}, {
'type': 'INTEGER',
'name': 'known_crashes',
'mode': 'NULLABLE'
}, {
'type': 'STRING',
'name': 'fuzzer',
'mode': 'NULLABLE'
}, {
'type': 'STRING',
'name': 'kind',
'mode': 'NULLABLE'
}]
},
}
}
},
projectId='test-clusterfuzz'),
mock.call().execute(),
mock.call(
body={
'configuration': {
'load': {
'destinationTable': {
'projectId': 'test-clusterfuzz',
'tableId': 'TestcaseRun$20160907',
'datasetId': 'fuzzer_stats'
},
'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
'writeDisposition':
'WRITE_TRUNCATE',
'sourceUris': [
'gs://test-bigquery-bucket/fuzzer/TestcaseRun/'
'date/20160907/*.json'
],
'sourceFormat':
'NEWLINE_DELIMITED_JSON',
}
}
},
projectId='test-clusterfuzz'),
mock.call().execute(),
],
# Otherwise we need to mock two calls to mock.call().execute().__str__()
# which does not seem to work well.
any_order=True)
