scripts/generate_code.py

#!/usr/bin/env python3
#
# Copyright 2021 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import filecmp
import glob
from pathlib import Path
import shutil
import subprocess
import generate_grpc_examples
from util import args, flatc, flatc_path, root_path, tests_path

# Specify the other paths that will be referenced
swift_code_gen = Path(root_path, "tests/swift/fuzzer/CodeGenerationTests")
ts_code_gen = Path(root_path, "tests/ts")
samples_path = Path(root_path, "samples")
reflection_path = Path(root_path, "reflection")


# Generate the code for flatbuffers reflection schema
def flatc_reflection(options, location, target):
  full_options = ["--no-prefix"] + options
  temp_dir = ".tmp"
  flatc(
      full_options,
      prefix=temp_dir,
      schema="reflection.fbs",
      cwd=reflection_path,
  )
  new_reflection_path = Path(reflection_path, temp_dir, target)
  original_reflection_path = Path(root_path, location, target)
  if not filecmp.cmp(str(new_reflection_path), str(original_reflection_path)):
    shutil.rmtree(str(original_reflection_path), ignore_errors=True)
    shutil.move(str(new_reflection_path), str(original_reflection_path))
  shutil.rmtree(str(Path(reflection_path, temp_dir)))


def flatc_annotate(schema, file, include=None, cwd=tests_path):
  cmd = [str(flatc_path)]
  if include:
    cmd += ["-I"] + [include]
  cmd += ["--annotate", schema, file]
  result = subprocess.run(cmd, cwd=str(cwd), check=True)


# Glob a pattern relative to file path
def glob(path, pattern):
  return [str(p) for p in path.glob(pattern)]


# flatc options that are shared
BASE_OPTS = ["--reflect-names", "--gen-mutable", "--gen-object-api"]
NO_INCL_OPTS = BASE_OPTS + ["--no-includes"]

# Language specific options
CS_OPTS = ["--csharp", "--cs-gen-json-serializer"]
CPP_OPTS = [
    "--cpp",
    "--gen-compare",
    "--gen-absl-hash",
] + (["--cpp-std", "c++0x"] if args.cpp_0x else [])

CPP_17_OPTS = NO_INCL_OPTS + [
    "--cpp",
    "--cpp-std",
    "c++17",
    "--cpp-static-reflection",
    "--gen-object-api",
]
RUST_OPTS = BASE_OPTS + [
    "--rust",
    "--gen-all",
    "--gen-name-strings",
    "--rust-module-root-file",
]
RUST_STANDALONE_OPTS = BASE_OPTS + [
    "--rust",
    "--gen-all",
    "--gen-name-strings",
]
RUST_SERIALIZE_OPTS = BASE_OPTS + [
    "--rust",
    "--gen-all",
    "--gen-name-strings",
    "--rust-serialize",
    "--rust-module-root-file",
]
TS_OPTS = ["--ts", "--gen-name-strings"]
LOBSTER_OPTS = ["--lobster"]
SWIFT_OPTS = ["--swift", "--gen-json-emit", "--bfbs-filenames", str(tests_path)]
SWIFT_OPTS_CODE_GEN = [
    "--swift",
    "--gen-json-emit",
    "--bfbs-filenames",
    str(swift_code_gen),
]
JAVA_OPTS = ["--java"]
KOTLIN_OPTS = ["--kotlin"]
PHP_OPTS = ["--php"]
DART_OPTS = ["--dart"]
PYTHON_OPTS = ["--python", "--python-typing", "--python-decode-obj-api-strings"]
BINARY_OPTS = ["-b", "--schema", "--bfbs-comments", "--bfbs-builtins"]
PROTO_OPTS = ["--proto"]

# Basic Usage

flatc(
    NO_INCL_OPTS
    + CPP_OPTS
    + CS_OPTS
    + [
        "--binary",
        "--java",
        "--kotlin",
        "--dart",
        "--go",
        "--lobster",
        "--php",
    ],
    schema="monster_test.fbs",
    include="include_test",
    data="monsterdata_test.json",
)

flatc(
    NO_INCL_OPTS + DART_OPTS,
    schema="include_test/include_test1.fbs",
    include="include_test/sub",
)

flatc(
    NO_INCL_OPTS + DART_OPTS,
    schema="include_test/sub/include_test2.fbs",
    include="include_test",
)

flatc(
    NO_INCL_OPTS + TS_OPTS,
    cwd=ts_code_gen,
    schema="../monster_test.fbs",
    include="../include_test",
    data="../monsterdata_test.json",
)

flatc(
    ["--lua", "--bfbs-filenames", str(tests_path)],
    schema="monster_test.fbs",
    include="include_test",
)

"""NOTE: The C++ gRPC golden is generated with the callback API enabled so that
the repository goldens exercise the callback client & server code paths.
If you need the legacy (non-callback) variant for comparison, invoke flatc
manually without --grpc-callback-api; we intentionally do not keep both to
minimize golden churn."""
flatc(
    NO_INCL_OPTS + CPP_OPTS + ["--grpc", "--grpc-callback-api"],
    schema="monster_test.fbs",
    include="include_test",
    data="monsterdata_test.json",
)

# Also generate a suffix variant exercising the callback API to keep prior
# *_generated naming convention in sync with new callback additions.
flatc(
    NO_INCL_OPTS
    + CPP_OPTS
    + ["--grpc", "--grpc-callback-api", "--filename-suffix", "_generated"],
    schema="monster_test.fbs",
    include="include_test",
)

flatc(
    RUST_OPTS,
    schema="monster_test.fbs",
    include="include_test",
    prefix="monster_test",
    data="monsterdata_test.json",
)

flatc(
    RUST_SERIALIZE_OPTS,
    schema="monster_test.fbs",
    include="include_test",
    prefix="monster_test_serialize",
    data="monsterdata_test.json",
)

flatc(
    options=BASE_OPTS + ["--python"],
    schema="monster_test.fbs",
    include="include_test",
    data="monsterdata_test.json",
)

flatc(
    options=BASE_OPTS + ["--python", "--gen-onefile"],
    schema="monster_test.fbs",
    include="include_test",
    data="monsterdata_test.json",
)

flatc(
    PROTO_OPTS,
    schema="prototest/test.proto",
)

# For Rust we currently generate two independent schemas, with namespace_test2
# duplicating the types in namespace_test1
flatc(
    RUST_OPTS + CS_OPTS,
    prefix="namespace_test",
    schema=[
        "namespace_test/namespace_test1.fbs",
        "namespace_test/namespace_test2.fbs",
    ],
)

flatc(
    [
        "--cpp",
        "--reflect-names",
        "--no-includes",
        "--gen-mutable",
        "--gen-object-api",
        "--gen-compare",
        "--gen-name-strings",
    ],
    prefix="namespace_test",
    schema=[
        "namespace_test/namespace_test1.fbs",
        "namespace_test/namespace_test2.fbs",
    ],
)

flatc(
    BASE_OPTS + CPP_OPTS + ["--cpp-ptr-type", "naked"],
    prefix="vector_table_naked_ptr",
    schema="vector_table_naked_ptr.fbs",
)

flatc(
    BASE_OPTS + CPP_OPTS + CS_OPTS + JAVA_OPTS + KOTLIN_OPTS + PHP_OPTS,
    prefix="union_vector",
    schema="union_vector/union_vector.fbs",
)

flatc(
    BASE_OPTS + TS_OPTS,
    cwd=ts_code_gen,
    prefix="union_vector",
    schema="../union_vector/union_vector.fbs",
)

flatc(
    BASE_OPTS + TS_OPTS + ["--gen-name-strings", "--gen-mutable"],
    cwd=ts_code_gen,
    include="../include_test",
    schema="../monster_test.fbs",
)

flatc(
    BASE_OPTS + TS_OPTS + ["-b"],
    cwd=ts_code_gen,
    include="../include_test",
    schema="../monster_test.fbs",
    data="../unicode_test.json",
)

flatc(
    BASE_OPTS + TS_OPTS + ["--gen-name-strings"],
    cwd=ts_code_gen,
    prefix="union_vector",
    schema="../union_vector/union_vector.fbs",
)

flatc(
    RUST_OPTS,
    prefix="include_test1",
    include="include_test",
    schema="include_test/include_test1.fbs",
)

flatc(
    RUST_STANDALONE_OPTS,
    include="include_test",
    schema="include_test/include_test1.fbs",
)

flatc(
    RUST_OPTS,
    prefix="include_test2",
    include="include_test",
    schema="include_test/sub/include_test2.fbs",
)

flatc(
    RUST_STANDALONE_OPTS,
    include="include_test",
    schema="include_test/sub/include_test2.fbs",
)

flatc(
    BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
    include="include_test",
    schema="monster_test.fbs",
)

# Generate the annotated binary of the monster_test binary schema.
flatc_annotate(
    schema="../reflection/reflection.fbs",
    file="monster_test.bfbs",
    include="include_test",
)

flatc_annotate(
    schema="monster_test.fbs",
    file="monsterdata_test.mon",
    include="include_test",
)

flatc(
    CPP_OPTS
    + NO_INCL_OPTS
    + [
        "--bfbs-comments",
        "--bfbs-builtins",
        "--bfbs-gen-embed",
        "--bfbs-filenames",
        str(tests_path),
    ],
    include="include_test",
    schema="monster_test.fbs",
)

flatc(
    BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
    include="include_test",
    schema="arrays_test.fbs",
)

flatc(
    ["--jsonschema", "--schema"],
    include="include_test",
    schema="monster_test.fbs",
)

if not args.skip_monster_extra:
  flatc(
      CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + KOTLIN_OPTS + PYTHON_OPTS,
      schema="monster_extra.fbs",
      data="monsterdata_extra.json",
  )

  flatc(
      DART_OPTS + ["--gen-object-api"],
      schema="monster_extra.fbs",
  )

flatc(
    CPP_OPTS
    + CS_OPTS
    + NO_INCL_OPTS
    + JAVA_OPTS
    + ["--jsonschema", "--scoped-enums"],
    schema="arrays_test.fbs",
)

flatc(
    ["--cpp", "--gen-compare", "--gen-mutable", "--gen-object-api", "--reflect-names"],
    schema="native_type_test.fbs",
)

flatc(
    [
        "--cpp",
        "--gen-mutable",
        "--gen-compare",
        "--gen-object-api",
        "--reflect-names",
    ],
    schema="native_inline_table_test.fbs",
)

flatc(
    ["--cpp", "--gen-compare", "--gen-mutable", "--gen-object-api", "--reflect-names"],
    schema="cross_namespace_pack_test.fbs",
)

flatc(
    RUST_OPTS,
    prefix="arrays_test",
    schema="arrays_test.fbs",
)

flatc(
    RUST_OPTS,
    prefix="rust_namer_test",
    schema="rust_namer_test.fbs",
)

flatc(
    BASE_OPTS + PYTHON_OPTS,
    schema="arrays_test.fbs",
)


flatc(
    BASE_OPTS + PYTHON_OPTS,
    schema="nested_union_test.fbs",
)

flatc(
    ["--python", "--gen-object-api"],
    schema="union_name_test.fbs",
)

flatc(
    NO_INCL_OPTS + CPP_OPTS,
    schema="default_vectors_strings_test.fbs",
)

# Optional Scalars
optional_scalars_schema = "optional_scalars.fbs"
flatc(["--java", "--kotlin", "--lobster"], schema=optional_scalars_schema)
flatc(TS_OPTS, cwd=ts_code_gen, schema="../optional_scalars.fbs")

flatc(
    ["--csharp", "--python", "--gen-object-api"], schema=optional_scalars_schema
)

flatc(RUST_OPTS, prefix="optional_scalars", schema=optional_scalars_schema)

flatc(NO_INCL_OPTS + CPP_OPTS, schema=optional_scalars_schema)

# Type / field collsion
type_field_collsion_schema = "type_field_collsion.fbs"

flatc(["--csharp", "--gen-object-api"], schema=type_field_collsion_schema)

# Union / value collision
flatc(
    CS_OPTS + ["--gen-object-api", "--gen-onefile"],
    prefix="union_value_collsion",
    schema="union_value_collision.fbs",
)

# Generate string/vector default code for tests
flatc(RUST_OPTS, prefix="more_defaults", schema="more_defaults.fbs")

# Generate the schema evolution tests
flatc(
    CPP_OPTS + ["--scoped-enums"],
    prefix="evolution_test",
    schema=glob(tests_path, "evolution_test/evolution_v*.fbs"),
)

# Generate the keywords tests
flatc(BASE_OPTS + CS_OPTS, schema="keyword_test.fbs")
flatc(RUST_OPTS, prefix="keyword_test", schema="keyword_test.fbs")
flatc(
    BASE_OPTS + CS_OPTS + ["--cs-global-alias", "--gen-onefile"],
    prefix="nested_namespace_test",
    schema=glob(tests_path, "nested_namespace_test/nested_namespace_test*.fbs"),
)
flatc(BASE_OPTS + DART_OPTS, prefix="../dart/test/", schema="keyword_test.fbs")

# Field key lookup with default value test
dictionary_lookup_schema = "dictionary_lookup.fbs"
flatc(["--java", "--kotlin"], schema=dictionary_lookup_schema)

# Swift Tests
swift_prefix = "swift/Tests/Flatbuffers"
flatc(
    SWIFT_OPTS + BASE_OPTS + ["--grpc"],
    schema="monster_test.fbs",
    include="include_test",
    prefix=swift_prefix,
)

flatc(
    SWIFT_OPTS + BASE_OPTS,
    schema="arrays_test.fbs",
    prefix=swift_prefix,
)

flatc(
    SWIFT_OPTS + BASE_OPTS,
    schema="union_vector/union_vector.fbs",
    prefix=swift_prefix,
)
flatc(SWIFT_OPTS, schema="optional_scalars.fbs", prefix=swift_prefix)
flatc(SWIFT_OPTS, schema="vector_has_test.fbs", prefix=swift_prefix)
flatc(SWIFT_OPTS, schema="nan_inf_test.fbs", prefix=swift_prefix)
flatc(
    SWIFT_OPTS + ["--gen-object-api"],
    schema="more_defaults.fbs",
    prefix=swift_prefix,
)
flatc(
    SWIFT_OPTS + BASE_OPTS,
    schema="MutatingBool.fbs",
    prefix=swift_prefix,
)

flatc(
    SWIFT_OPTS_CODE_GEN + BASE_OPTS + ["--grpc", "--swift-implementation-only"],
    schema="test_import.fbs",
    cwd=swift_code_gen,
)

flatc(
    SWIFT_OPTS_CODE_GEN + NO_INCL_OPTS + ["--grpc"],
    schema="test_no_include.fbs",
    cwd=swift_code_gen,
)

flatc(
    SWIFT_OPTS_CODE_GEN + BASE_OPTS,
    schema="empty_vtable.fbs",
    cwd=swift_code_gen,
    prefix="../../Tests/Flatbuffers/",
)

# Swift Wasm Tests
swift_Wasm_prefix = "swift/Wasm.tests/Tests/FlatBuffers.Test.Swift.WasmTests"
flatc(
    SWIFT_OPTS + BASE_OPTS,
    schema="monster_test.fbs",
    include="include_test",
    prefix=swift_Wasm_prefix,
)

# Nim Tests
NIM_OPTS = BASE_OPTS + ["--nim"]
flatc(NIM_OPTS, schema="monster_test.fbs", include="include_test")
flatc(NIM_OPTS, schema="optional_scalars.fbs")
flatc(NIM_OPTS, schema="more_defaults.fbs")
flatc(NIM_OPTS, schema="MutatingBool.fbs")

# --filename-suffix and --filename-ext tests
flatc(
    CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-ext", "hpp"],
    include="include_test",
    prefix="monster_test_suffix/ext_only",
    schema="monster_test.fbs",
)
flatc(
    CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-suffix", "_suffix"],
    include="include_test",
    prefix="monster_test_suffix/filesuffix_only",
    schema="monster_test.fbs",
)
flatc(
    CPP_OPTS
    + NO_INCL_OPTS
    + ["--grpc", "--filename-suffix", "_suffix", "--filename-ext", "hpp"],
    include="include_test",
    prefix="monster_test_suffix",
    schema="monster_test.fbs",
)

# Flag c++17 requires Clang6, GCC7, MSVC2017 (_MSC_VER >= 1914) or higher.
cpp_17_prefix = "cpp17/generated_cpp17"
flatc(
    CPP_17_OPTS,
    schema="monster_test.fbs",
    include="include_test",
    prefix=cpp_17_prefix,
)
flatc(
    CPP_17_OPTS,
    schema="optional_scalars.fbs",
    prefix=cpp_17_prefix,
)
flatc(
    CPP_17_OPTS,
    schema="union_vector/union_vector.fbs",
    prefix=cpp_17_prefix,
)

# Private annotations
annotations_test_schema = "private_annotation_test.fbs"

flatc(
    RUST_OPTS + ["--no-leak-private-annotation", "--gen-object-api"],
    prefix="private_annotation_test",
    schema=annotations_test_schema,
)

# Sample files
samples_schema = "monster.fbs"
flatc(
    BASE_OPTS + CPP_OPTS + LOBSTER_OPTS + SWIFT_OPTS,
    schema=samples_schema,
    cwd=samples_path,
)
flatc(
    RUST_OPTS, prefix="rust_generated", schema=samples_schema, cwd=samples_path
)
flatc(
    BINARY_OPTS + ["--bfbs-filenames", str(samples_path)],
    schema=samples_schema,
    cwd=samples_path,
)

# Reflection

# Skip generating the reflection if told too, as we run this script after
# building flatc which uses the reflection_generated.h itself.
if not args.skip_gen_reflection:
  # C++ Reflection
  flatc_reflection(
      ["-c", "--cpp-std", "c++0x"],
      "include/flatbuffers",
      "reflection_generated.h",
  )

# Python Reflection
flatc_reflection(["-p"], "python/flatbuffers", "reflection")

# Java Reflection
flatc_reflection(
    ["-j", "--java-package-prefix", "com.google.flatbuffers"],
    "java/src/main/java",
    "com/google/flatbuffers/reflection",
)

# Annotation


def flatc_annotate(schema, include=None, data=None, cwd=tests_path):
  cmd = [str(flatc_path)]
  if include:
    cmd += ["-I"] + [include]
  cmd += ["--annotate", schema]
  if data:
    cmd += [data] if isinstance(data, str) else data
  subprocess.run(cmd, cwd=str(cwd), check=True)


flatc_annotate(
    schema="monster_test.fbs",
    include="include_test",
    data="monsterdata_test.mon",
)

# Run the generate_grpc_examples script
generate_grpc_examples.GenerateGRPCExamples()