Python google.protobuf.text_format.ParseError() Examples

def testConsumeOctalIntegers(self):
    """Test support for C style octal integers."""
    text = '00 -00 04 0755 -010 007 -0033 08 -09 01'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertEqual(0, tokenizer.ConsumeInteger())
    self.assertEqual(0, tokenizer.ConsumeInteger())
    self.assertEqual(4, tokenizer.ConsumeInteger())
    self.assertEqual(0o755, tokenizer.ConsumeInteger())
    self.assertEqual(-0o10, tokenizer.ConsumeInteger())
    self.assertEqual(7, tokenizer.ConsumeInteger())
    self.assertEqual(-0o033, tokenizer.ConsumeInteger())
    with self.assertRaises(text_format.ParseError):
      tokenizer.ConsumeInteger()  # 08
    tokenizer.NextToken()
    with self.assertRaises(text_format.ParseError):
      tokenizer.ConsumeInteger()  # -09
    tokenizer.NextToken()
    self.assertEqual(1, tokenizer.ConsumeInteger())
    self.assertTrue(tokenizer.AtEnd()) 

def testConsumeByteString(self):
    text = '"string1\''
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = 'string1"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\xt"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\x"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 

def testConsumeAndCheckTrailingComment(self):
    text = 'some_number: 4  # some comment'  # trailing comment on the same line
    tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False)
    self.assertRaises(text_format.ParseError,
                      tokenizer.ConsumeCommentOrTrailingComment)

    self.assertEqual('some_number', tokenizer.ConsumeIdentifier())
    self.assertEqual(tokenizer.token, ':')
    tokenizer.NextToken()
    self.assertRaises(text_format.ParseError,
                      tokenizer.ConsumeCommentOrTrailingComment)
    self.assertEqual(4, tokenizer.ConsumeInteger())
    self.assertFalse(tokenizer.AtEnd())

    self.assertEqual((True, '# some comment'),
                     tokenizer.ConsumeCommentOrTrailingComment())
    self.assertTrue(tokenizer.AtEnd()) 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def get_proto(push_list, output_dir):
    bench_factory = aibench_pb2.BenchFactory()
    model_factory = aibench_pb2.ModelFactory()
    try:
        with open("aibench/proto/benchmark.meta", "rb") as fin:
            file_content = fin.read()
            text_format.Parse(file_content, bench_factory)
            filepath = output_dir + "/benchmark.pb"
            with open(filepath, "wb") as fout:
                fout.write(bench_factory.SerializeToString())
                push_list.append(filepath)
        with open("aibench/proto/model.meta", "rb") as fin:
            file_content = fin.read()
            text_format.Parse(file_content, model_factory)
            filepath = output_dir + "/model.pb"
            with open(filepath, "wb") as fout:
                fout.write(model_factory.SerializeToString())
                push_list.append(filepath)
    except text_format.ParseError as e:
        raise IOError("Cannot parse file.", e)

    return bench_factory, model_factory 

def testConsumeByteString(self):
    text = '"string1\''
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = 'string1"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\xt"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\x"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def testConsumeByteString(self):
    text = '"string1\''
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = 'string1"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\xt"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)

    text = '\n"\\x"'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def load_labelmap(path):
  """Loads label map proto.

  Args:
    path: path to StringIntLabelMap proto text file.
  Returns:
    a StringIntLabelMapProto
  """
  with tf.gfile.GFile(path, 'r') as fid:
    label_map_string = fid.read()
    label_map = string_int_label_map_pb2.StringIntLabelMap()
    try:
      text_format.Merge(label_map_string, label_map)
    except text_format.ParseError:
      label_map.ParseFromString(label_map_string)
  _validate_label_map(label_map)
  return label_map 

def testParseInvalidUtf8(self, message_module):
    message = message_module.TestAllTypes()
    text = 'repeated_string: "\\xc3\\xc3"'
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(e.exception.GetLine(), 1)
    self.assertEqual(e.exception.GetColumn(), 28) 

def testParseSingleWord(self, message_module):
    message = message_module.TestAllTypes()
    text = 'foo'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"foo".'), text_format.Parse, text, message) 

def testParseUnknownField(self, message_module):
    message = message_module.TestAllTypes()
    text = 'unknown_field: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"unknown_field".'), text_format.Parse, text, message)
    text = ('optional_int32: 123\n'
            'unknown_field: 8\n'
            'optional_nested_message { bb: 45 }')
    text_format.Parse(text, message, allow_unknown_field=True)
    self.assertEqual(message.optional_nested_message.bb, 45)
    self.assertEqual(message.optional_int32, 123) 

def testParseBadEnumValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_nested_enum: BARR'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : \'optional_nested_enum: BARR\': '
                           r'Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value named BARR.'), text_format.Parse,
                          text, message) 

def testParseBadIntValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_int32: bork'
    six.assertRaisesRegex(self, text_format.ParseError,
                          ('1:17 : \'optional_int32: bork\': '
                           'Couldn\'t parse integer: bork'),
                          text_format.Parse, text, message) 

def testParseDuplicateMessages(self, message_module):
    message = message_module.TestAllTypes()
    text = ('optional_nested_message { bb: 1 } '
            'optional_nested_message { bb: 2 }')
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:59 : Message type "\w+.TestAllTypes" '
        r'should not have multiple "optional_nested_message" fields.'),
                          text_format.Parse, text,
                          message) 

def testParseDuplicateScalars(self, message_module):
    message = message_module.TestAllTypes()
    text = ('optional_int32: 42 ' 'optional_int32: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:36 : Message type "\w+.TestAllTypes" should not '
        r'have multiple "optional_int32" fields.'), text_format.Parse, text,
                          message) 

def testParseMessageByFieldNumber(self):
    message = unittest_pb2.TestAllTypes()
    text = ('34: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message, allow_field_number=True)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('1 {\n'
            '  1545008 {\n'
            '    15: 23\n'
            '  }\n'
            '  1547769 {\n'
            '    25: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message, allow_field_number=True)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str)

    # Can't parse field number without set allow_field_number=True.
    message = unittest_pb2.TestAllTypes()
    text = '34:1\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"34".'), text_format.Parse, text, message)

    # Can't parse if field number is not found.
    text = '1234:1\n'
    six.assertRaisesRegex(
        self,
        text_format.ParseError,
        (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
         r'"1234".'),
        text_format.Parse,
        text,
        message,
        allow_field_number=True) 

def testParseBadIdentifier(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { "bb": 1 }')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(str(e.exception),
                     '1:27 : \'optional_nested_message { "bb": 1 }\': '
                     'Expected identifier or number, got "bb".') 

def testParseBadExtension(self):
    message = unittest_pb2.TestAllExtensions()
    text = '[unknown_extension]: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError,
                          '1:2 : Extension "unknown_extension" not registered.',
                          text_format.Parse, text, message)
    message = unittest_pb2.TestAllTypes()
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
        'extensions.'), text_format.Parse, text, message) 

def testParseDuplicateExtensionScalars(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_int32_extension]: 42 '
            '[protobuf_unittest.optional_int32_extension]: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:96 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_int32_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseDuplicateExtensionMessages(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_nested_message_extension]: {} '
            '[protobuf_unittest.optional_nested_message_extension]: {}')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:114 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_nested_message_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseGroupNotClosed(self):
    message = unittest_pb2.TestAllTypes()
    text = 'RepeatedGroup: <'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected ">".',
                          text_format.Parse, text, message)
    text = 'RepeatedGroup: {'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected "}".',
                          text_format.Parse, text, message) 

def testMergeExpandedAnyDescriptorPoolMissingType(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n'
            '  }\n'
            '}\n')
    with self.assertRaises(text_format.ParseError) as e:
      empty_pool = descriptor_pool.DescriptorPool()
      text_format.Merge(text, message, descriptor_pool=empty_pool)
    self.assertEqual(
        str(e.exception),
        'Type protobuf_unittest.OneString not found in descriptor pool') 

def testMergeMissingAnyEndToken(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Merge(text, message)
    self.assertEqual(str(e.exception), '3:11 : Expected "}".') 

def testConsumeIntegers(self):
    # This test only tests the failures in the integer parsing methods as well
    # as the '0' special cases.
    int64_max = (1 << 63) - 1
    uint32_max = (1 << 32) - 1
    text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint64, tokenizer)
    self.assertEqual(-1, text_format._ConsumeInt32(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt32, tokenizer)
    self.assertEqual(uint32_max + 1, text_format._ConsumeInt64(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt64, tokenizer)
    self.assertEqual(int64_max + 1, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd())

    text = '-0 -0 0 0'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd()) 

def testConsumeBool(self):
    text = 'not-a-bool'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) 

def testSkipComment(self):
    tokenizer = text_format.Tokenizer('# some comment'.splitlines())
    self.assertTrue(tokenizer.AtEnd())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment) 

def testParseInvalidUtf8(self, message_module):
    message = message_module.TestAllTypes()
    text = 'repeated_string: "\\xc3\\xc3"'
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(e.exception.GetLine(), 1)
    self.assertEqual(e.exception.GetColumn(), 28) 

def testParseSingleWord(self, message_module):
    message = message_module.TestAllTypes()
    text = 'foo'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"foo".'), text_format.Parse, text, message) 

def testParseUnknownField(self, message_module):
    message = message_module.TestAllTypes()
    text = 'unknown_field: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"unknown_field".'), text_format.Parse, text, message) 

def testParseBadEnumValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_nested_enum: BARR'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value named BARR.'), text_format.Parse,
                          text, message) 

def testParseBadIntValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_int32: bork'
    six.assertRaisesRegex(self, text_format.ParseError,
                          ('1:17 : Couldn\'t parse integer: bork'),
                          text_format.Parse, text, message) 

def testParseMessageByFieldNumber(self):
    message = unittest_pb2.TestAllTypes()
    text = ('34: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message, allow_field_number=True)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('1 {\n'
            '  1545008 {\n'
            '    15: 23\n'
            '  }\n'
            '  1547769 {\n'
            '    25: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message, allow_field_number=True)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str)

    # Can't parse field number without set allow_field_number=True.
    message = unittest_pb2.TestAllTypes()
    text = '34:1\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"34".'), text_format.Parse, text, message)

    # Can't parse if field number is not found.
    text = '1234:1\n'
    six.assertRaisesRegex(
        self,
        text_format.ParseError,
        (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
         r'"1234".'),
        text_format.Parse,
        text,
        message,
        allow_field_number=True) 

def testParseBadIdentifier(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { "bb": 1 }')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(str(e.exception),
                     '1:27 : Expected identifier or number, got "bb".') 

def testParseBadExtension(self):
    message = unittest_pb2.TestAllExtensions()
    text = '[unknown_extension]: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError,
                          '1:2 : Extension "unknown_extension" not registered.',
                          text_format.Parse, text, message)
    message = unittest_pb2.TestAllTypes()
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
        'extensions.'), text_format.Parse, text, message) 

def testParseNumericUnknownEnum(self):
    message = unittest_pb2.TestAllTypes()
    text = 'optional_nested_enum: 100'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value with number 100.'), text_format.Parse,
                          text, message) 

def testParseDuplicateExtensionScalars(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_int32_extension]: 42 '
            '[protobuf_unittest.optional_int32_extension]: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:96 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_int32_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseDuplicateExtensionMessages(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_nested_message_extension]: {} '
            '[protobuf_unittest.optional_nested_message_extension]: {}')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:114 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_nested_message_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseDuplicateScalars(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_int32: 42 ' 'optional_int32: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
        'have multiple "optional_int32" fields.'), text_format.Parse, text,
                          message) 

def testParseGroupNotClosed(self):
    message = unittest_pb2.TestAllTypes()
    text = 'RepeatedGroup: <'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected ">".',
                          text_format.Parse, text, message)
    text = 'RepeatedGroup: {'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected "}".',
                          text_format.Parse, text, message) 

def testMergeExpandedAnyDescriptorPoolMissingType(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n'
            '  }\n'
            '}\n')
    with self.assertRaises(text_format.ParseError) as e:
      empty_pool = descriptor_pool.DescriptorPool()
      text_format.Merge(text, message, descriptor_pool=empty_pool)
    self.assertEqual(
        str(e.exception),
        'Type protobuf_unittest.OneString not found in descriptor pool') 

def testMergeMissingAnyEndToken(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Merge(text, message)
    self.assertEqual(str(e.exception), '3:11 : Expected "}".') 

def testConsumeIntegers(self):
    # This test only tests the failures in the integer parsing methods as well
    # as the '0' special cases.
    int64_max = (1 << 63) - 1
    uint32_max = (1 << 32) - 1
    text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint64, tokenizer)
    self.assertEqual(-1, text_format._ConsumeInt32(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt32, tokenizer)
    self.assertEqual(uint32_max + 1, text_format._ConsumeInt64(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt64, tokenizer)
    self.assertEqual(int64_max + 1, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd())

    text = '-0 -0 0 0'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd()) 

def testConsumeBool(self):
    text = 'not-a-bool'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) 

def testSkipComment(self):
    tokenizer = text_format.Tokenizer('# some comment'.splitlines())
    self.assertTrue(tokenizer.AtEnd())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment) 

def testConsumeTrailingComment(self):
    text = 'some_number: 4\n# some comment'
    tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False)
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment)

    self.assertEqual('some_number', tokenizer.ConsumeIdentifier())
    self.assertEqual(tokenizer.token, ':')
    tokenizer.NextToken()
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment)
    self.assertEqual(4, tokenizer.ConsumeInteger())
    self.assertFalse(tokenizer.AtEnd())

    self.assertEqual('# some comment', tokenizer.ConsumeComment())
    self.assertTrue(tokenizer.AtEnd()) 

def testParseInvalidUtf8(self, message_module):
    message = message_module.TestAllTypes()
    text = 'repeated_string: "\\xc3\\xc3"'
    self.assertRaises(text_format.ParseError, text_format.Parse, text, message) 

def testParseSingleWord(self, message_module):
    message = message_module.TestAllTypes()
    text = 'foo'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"foo".'), text_format.Parse, text, message) 

def testParseUnknownField(self, message_module):
    message = message_module.TestAllTypes()
    text = 'unknown_field: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"unknown_field".'), text_format.Parse, text, message) 

def testParseBadEnumValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_nested_enum: BARR'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value named BARR.'), text_format.Parse,
                          text, message)

    message = message_module.TestAllTypes()
    text = 'optional_nested_enum: 100'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value with number 100.'), text_format.Parse,
                          text, message) 

def testParseBadIntValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_int32: bork'
    six.assertRaisesRegex(self, text_format.ParseError,
                          ('1:17 : Couldn\'t parse integer: bork'),
                          text_format.Parse, text, message) 

def testParseMessageByFieldNumber(self):
    message = unittest_pb2.TestAllTypes()
    text = ('34: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message, allow_field_number=True)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('1 {\n'
            '  1545008 {\n'
            '    15: 23\n'
            '  }\n'
            '  1547769 {\n'
            '    25: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message, allow_field_number=True)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str)

    # Can't parse field number without set allow_field_number=True.
    message = unittest_pb2.TestAllTypes()
    text = '34:1\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"34".'), text_format.Parse, text, message)

    # Can't parse if field number is not found.
    text = '1234:1\n'
    six.assertRaisesRegex(
        self,
        text_format.ParseError,
        (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
         r'"1234".'),
        text_format.Parse,
        text,
        message,
        allow_field_number=True) 

def testParseBadExtension(self):
    message = unittest_pb2.TestAllExtensions()
    text = '[unknown_extension]: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError,
                          '1:2 : Extension "unknown_extension" not registered.',
                          text_format.Parse, text, message)
    message = unittest_pb2.TestAllTypes()
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
        'extensions.'), text_format.Parse, text, message) 

def testParseDuplicateExtensionScalars(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_int32_extension]: 42 '
            '[protobuf_unittest.optional_int32_extension]: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:96 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_int32_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseDuplicateNestedMessageScalars(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { bb: 1 } '
            'optional_nested_message { bb: 2 }')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
        'should not have multiple "bb" fields.'), text_format.Parse, text,
                          message) 

def testParseDuplicateScalars(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_int32: 42 ' 'optional_int32: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
        'have multiple "optional_int32" fields.'), text_format.Parse, text,
                          message) 

def testMergeExpandedAnyNoDescriptorPool(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n'
            '  }\n'
            '}\n')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Merge(text, message, descriptor_pool=None)
    self.assertEqual(str(e.exception),
                     'Descriptor pool required to parse expanded Any field') 

def testMergeExpandedAnyDescriptorPoolMissingType(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n'
            '  }\n'
            '}\n')
    with self.assertRaises(text_format.ParseError) as e:
      empty_pool = descriptor_pool.DescriptorPool()
      text_format.Merge(text, message, descriptor_pool=empty_pool)
    self.assertEqual(
        str(e.exception),
        'Type protobuf_unittest.OneString not found in descriptor pool') 

def testConsumeIntegers(self):
    # This test only tests the failures in the integer parsing methods as well
    # as the '0' special cases.
    int64_max = (1 << 63) - 1
    uint32_max = (1 << 32) - 1
    text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint64, tokenizer)
    self.assertEqual(-1, text_format._ConsumeInt32(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeUint32, tokenizer)
    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt32, tokenizer)
    self.assertEqual(uint32_max + 1, text_format._ConsumeInt64(tokenizer))

    self.assertRaises(text_format.ParseError,
                      text_format._ConsumeInt64, tokenizer)
    self.assertEqual(int64_max + 1, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd())

    text = '-0 -0 0 0'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint32(tokenizer))
    self.assertEqual(0, text_format._ConsumeUint64(tokenizer))
    self.assertTrue(tokenizer.AtEnd()) 

def testConsumeBool(self):
    text = 'not-a-bool'
    tokenizer = text_format.Tokenizer(text.splitlines())
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) 

def testConsumeTrailingComment(self):
    text = 'some_number: 4\n# some comment'
    tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False)
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment)

    self.assertEqual('some_number', tokenizer.ConsumeIdentifier())
    self.assertEqual(tokenizer.token, ':')
    tokenizer.NextToken()
    self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment)
    self.assertEqual(4, tokenizer.ConsumeInteger())
    self.assertFalse(tokenizer.AtEnd())

    self.assertEqual('# some comment', tokenizer.ConsumeComment())
    self.assertTrue(tokenizer.AtEnd()) 

def _read_file(filename):
  """Reads a file containing `GraphDef` and returns the protocol buffer.

  Args:
    filename: `graph_def` filename including the path.

  Returns:
    A `GraphDef` protocol buffer.

  Raises:
    IOError: If the file doesn't exist, or cannot be successfully parsed.
  """
  graph_def = graph_pb2.GraphDef()
  if not file_io.file_exists(filename):
    raise IOError("File %s does not exist." % filename)
  # First try to read it as a binary file.
  file_content = file_io.read_file_to_string(filename)
  try:
    graph_def.ParseFromString(file_content)
    return graph_def
  except Exception:  # pylint: disable=broad-except
    pass

  # Next try to read it as a text file.
  try:
    text_format.Merge(file_content.decode("utf-8"), graph_def)
  except text_format.ParseError as e:
    raise IOError("Cannot parse file %s: %s." % (filename, str(e)))

  return graph_def 

def read_meta_graph_file(filename):
  """Reads a file containing `MetaGraphDef` and returns the protocol buffer.

  Args:
    filename: `meta_graph_def` filename including the path.

  Returns:
    A `MetaGraphDef` protocol buffer.

  Raises:
    IOError: If the file doesn't exist, or cannot be successfully parsed.
  """
  meta_graph_def = meta_graph_pb2.MetaGraphDef()
  if not file_io.file_exists(filename):
    raise IOError("File %s does not exist." % filename)
  # First try to read it as a binary file.
  file_content = file_io.read_file_to_string(filename)
  try:
    meta_graph_def.ParseFromString(file_content)
    return meta_graph_def
  except Exception:  # pylint: disable=broad-except
    pass

  # Next try to read it as a text file.
  try:
    text_format.Merge(file_content.decode("utf-8"), meta_graph_def)
  except text_format.ParseError as e:
    raise IOError("Cannot parse file %s: %s." % (filename, str(e)))

  return meta_graph_def 

def testParseInvalidUtf8(self, message_module):
    message = message_module.TestAllTypes()
    text = 'repeated_string: "\\xc3\\xc3"'
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(e.exception.GetLine(), 1)
    self.assertEqual(e.exception.GetColumn(), 28) 

def testParseSingleWord(self, message_module):
    message = message_module.TestAllTypes()
    text = 'foo'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"foo".'), text_format.Parse, text, message) 

def testParseUnknownField(self, message_module):
    message = message_module.TestAllTypes()
    text = 'unknown_field: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"unknown_field".'), text_format.Parse, text, message)
    text = ('optional_int32: 123\n'
            'unknown_field: 8\n'
            'optional_nested_message { bb: 45 }')
    text_format.Parse(text, message, allow_unknown_field=True)
    self.assertEqual(message.optional_nested_message.bb, 45)
    self.assertEqual(message.optional_int32, 123) 

def testParseBadEnumValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_nested_enum: BARR'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : \'optional_nested_enum: BARR\': '
                           r'Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value named BARR.'), text_format.Parse,
                          text, message) 

def testParseBadIntValue(self, message_module):
    message = message_module.TestAllTypes()
    text = 'optional_int32: bork'
    six.assertRaisesRegex(self, text_format.ParseError,
                          ('1:17 : \'optional_int32: bork\': '
                           'Couldn\'t parse integer: bork'),
                          text_format.Parse, text, message) 

def testParseDuplicateMessages(self, message_module):
    message = message_module.TestAllTypes()
    text = ('optional_nested_message { bb: 1 } '
            'optional_nested_message { bb: 2 }')
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:59 : Message type "\w+.TestAllTypes" '
        r'should not have multiple "optional_nested_message" fields.'),
                          text_format.Parse, text,
                          message) 

def testParseDuplicateScalars(self, message_module):
    message = message_module.TestAllTypes()
    text = ('optional_int32: 42 ' 'optional_int32: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:36 : Message type "\w+.TestAllTypes" should not '
        r'have multiple "optional_int32" fields.'), text_format.Parse, text,
                          message) 

def testParseExistingScalarInMessage(self, message_module):
    message = message_module.TestAllTypes(optional_int32=42)
    text = 'optional_int32: 67'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'Message type "\w+.TestAllTypes" should not '
                           r'have multiple "optional_int32" fields.'),
                          text_format.Parse, text, message) 

def _parse_saved_model(export_dir):
  """Reads the savedmodel.pb or savedmodel.pbtxt file containing `SavedModel`.

  Args:
    export_dir: Directory containing the SavedModel file.

  Returns:
    A `SavedModel` protocol buffer.

  Raises:
    IOError: If the file does not exist, or cannot be successfully parsed.
  """
  # Build the path to the SavedModel in pbtxt format.
  path_to_pbtxt = os.path.join(
      compat.as_bytes(export_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT))
  # Build the path to the SavedModel in pb format.
  path_to_pb = os.path.join(
      compat.as_bytes(export_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))

  # Ensure that the SavedModel exists at either path.
  if not file_io.file_exists(path_to_pbtxt) and not file_io.file_exists(
      path_to_pb):
    raise IOError("SavedModel file does not exist at: %s" % export_dir)

  saved_model = saved_model_pb2.SavedModel()

  # Parse the SavedModel protocol buffer.
  try:
    file_content = file_io.read_file_to_string(path_to_pb)
    saved_model.ParseFromString(file_content)
    return saved_model
  except Exception:  # pylint: disable=broad-except
    # Pass for exceptions in order to try reading the file in text format.
    pass

  try:
    file_content = file_io.read_file_to_string(path_to_pbtxt)
    text_format.Merge(file_content.decode("utf-8"), saved_model)
  except text_format.ParseError as e:
    raise IOError("Cannot parse file %s: %s." % (path_to_pbtxt, str(e)))
  return saved_model 

def get_checkpoint_state(checkpoint_dir, latest_filename=None):
  """Returns CheckpointState proto from the "checkpoint" file.

  If the "checkpoint" file contains a valid CheckpointState
  proto, returns it.

  Args:
    checkpoint_dir: The directory of checkpoints.
    latest_filename: Optional name of the checkpoint file.  Default to
      'checkpoint'.

  Returns:
    A CheckpointState if the state was available, None
    otherwise.

  Raises:
    ValueError: if the checkpoint read doesn't have model_checkpoint_path set.
  """
  ckpt = None
  coord_checkpoint_filename = _GetCheckpointFilename(checkpoint_dir,
                                                     latest_filename)
  f = None
  try:
    # Check that the file exists before opening it to avoid
    # many lines of errors from colossus in the logs.
    if file_io.file_exists(coord_checkpoint_filename):
      file_content = file_io.read_file_to_string(
          coord_checkpoint_filename).decode("utf-8")
      ckpt = CheckpointState()
      text_format.Merge(file_content, ckpt)
      if not ckpt.model_checkpoint_path:
        raise ValueError("Invalid checkpoint state loaded from %s",
                         checkpoint_dir)
      # For relative model_checkpoint_path and all_model_checkpoint_paths,
      # prepend checkpoint_dir.
      if not os.path.isabs(ckpt.model_checkpoint_path):
        ckpt.model_checkpoint_path = os.path.join(checkpoint_dir,
                                                  ckpt.model_checkpoint_path)
      for i in range(len(ckpt.all_model_checkpoint_paths)):
        p = ckpt.all_model_checkpoint_paths[i]
        if not os.path.isabs(p):
          ckpt.all_model_checkpoint_paths[i] = os.path.join(checkpoint_dir, p)
  except errors.OpError as e:
    # It's ok if the file cannot be read
    logging.warning(str(e))
    logging.warning("%s: Checkpoint ignored", coord_checkpoint_filename)
    return None
  except text_format.ParseError as e:
    logging.warning(str(e))
    logging.warning("%s: Checkpoint ignored", coord_checkpoint_filename)
    return None
  finally:
    if f:
      f.close()
  return ckpt