diff --git a/.github/scripts/flatbuffers-on.py b/.github/scripts/flatbuffers-on.py new file mode 100644 index 0000000000..e860b5aa67 --- /dev/null +++ b/.github/scripts/flatbuffers-on.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run the focused FlatBuffers ON gates; never accept empty or skipped tests.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tarfile +import time +import xml.etree.ElementTree as ET + +# Keep the FlatBuffers pin synchronized with MODULE.bazel and WORKSPACE. +FLATBUFFERS_VERSION = "25.2.10" +FLATBUFFERS_SHA256 = "b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421" +GTEST_VERSION = "1.14.0" +GTEST_SHA256 = "8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7" +TESTS = {"brpc_flatbuffers_unittest": 22, "brpc_flatbuffers_protocol_unittest": 36} + + +def list_gtests(text): + names = set() + suite = None + for line in text.splitlines(): + value = line.split("#", 1)[0].strip() + if not value: + continue + if not line[0].isspace() and value.endswith("."): + suite = value + elif line[0].isspace() and suite and re.fullmatch(r"[A-Za-z0-9_/]+", value): + name = suite + value + if any(part.startswith("DISABLED_") for part in re.split(r"[./]", name)): + raise ValueError("Disabled test in ON gate: " + name) + if name in names: + raise ValueError("Duplicate discovered test: " + name) + names.add(name) + if not names: + raise ValueError("No GoogleTest cases discovered") + return names + + +def xml_cases(path): + path = Path(path) + if not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: + raise ValueError("Missing or oversized test XML: " + str(path)) + root = ET.parse(path).getroot() + for suite in root.iter(): + if suite.tag in ("testsuites", "testsuite"): + for attribute in ("failures", "errors", "disabled", "skipped"): + if int(suite.get(attribute, "0")) != 0: + raise ValueError("Nonzero " + attribute + " in " + str(path)) + cases = list(root.iter("testcase")) + if not cases: + raise ValueError("Test XML contains no cases: " + str(path)) + for case in cases: + if (case.find("failure") is not None or case.find("error") is not None or + case.find("skipped") is not None or case.get("status") == "notrun" or + case.get("result") in ("skipped", "suppressed")): + raise ValueError("Failed or unexecuted case: " + str(case.attrib)) + return root, cases + + +def verify_gtests(xml, listing, expected_count): + expected = list_gtests(Path(listing).read_text()) + root, cases = xml_cases(xml) + actual = [case.get("classname", "") + "." + case.get("name", "") for case in cases] + if len(expected) != expected_count or len(actual) != len(set(actual)) or set(actual) != expected: + raise ValueError("Executed cases differ from discovery or the expected count: " + str(xml)) + if int(root.get("tests", "-1")) != len(actual): + raise ValueError("Incorrect XML test count: " + str(xml)) + return {"executed": len(actual), "failed": 0, "skipped": 0} + + +def verify_ctest(xml, required): + _, cases = xml_cases(xml) + names = [case.get("name", "") for case in cases] + if len(names) != len(set(names)) or not set(required) <= set(names): + raise ValueError("Required CTest cases did not execute: " + str(required)) + return {"executed": len(names), "names": names, "failed": 0, "skipped": 0} + + +def extract_archive(archive, digest, destination, prefix): + archive = Path(archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != digest: + raise ValueError("Dependency checksum mismatch: " + str(archive)) + destination = Path(destination) + if destination.exists(): + raise ValueError("Refusing to overwrite dependency directory") + with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + if len(members) > 30000 or sum(item.size for item in members) > 512 * 1024 * 1024: + raise ValueError("Dependency archive exceeds extraction limits") + paths = set() + links = {} + for member in members: + path = PurePosixPath(member.name) + if (not path.parts or path.is_absolute() or ".." in path.parts or + path.parts[0] != prefix or path in paths or + not (member.isdir() or member.isfile() or member.issym())): + raise ValueError("Unsafe dependency archive member: " + member.name) + paths.add(path) + if member.issym(): + if len(path.parts) < 2: + raise ValueError("Archive root must not be a symlink") + links[path] = member + for path in paths: + if any(parent in links for parent in path.parents): + raise ValueError("Archive member traverses a symlink: " + str(path)) + for path, member in links.items(): + target = PurePosixPath(member.linkname) + if not member.linkname or target.is_absolute(): + raise ValueError("Unsafe archive symlink: " + member.name) + parts = list(path.parent.parts) + for part in target.parts: + if part == "..": + if len(parts) <= 1: + raise ValueError("Escaping archive symlink: " + member.name) + parts.pop() + else: + parts.append(part) + if not parts or parts[0] != prefix or PurePosixPath(*parts) in links: + raise ValueError("Escaping or chained archive symlink: " + member.name) + destination.mkdir(parents=True) + # Create links only after all regular data, so extraction never writes + # through a symlink. The pinned archives need no hardlinks/link chains. + source.extractall(destination, members=[member for member in members if not member.issym()]) + for path, member in links.items(): + link = destination / str(path) + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(member.linkname) + return destination / prefix + + +def check_enabled(header): + if not re.search(r"^\s*#\s*define\s+BRPC_WITH_FLATBUFFERS\s+1\s*$", + Path(header).read_text(), re.MULTILINE): + raise ValueError("FlatBuffers is not enabled in " + str(header)) + + +def signal_group(process, signum): + try: + os.killpg(process.pid, signum) + return True + except ProcessLookupError: + return False + + +def stop_process(process, grace=5): + # The group may outlive its leader; waiting for the leader alone is not + # sufficient when a compiler/test child ignores SIGTERM. + if signal_group(process, signal.SIGTERM): + try: + process.wait(timeout=grace) + except subprocess.TimeoutExpired: + pass + signal_group(process, signal.SIGKILL) + process.wait() + + +class Runner: + def __init__(self, args): + self.args = args + self.source = args.source.resolve(strict=True) + self.work = args.work.resolve() + if self.work == self.source or self.source in self.work.parents: + raise ValueError("--work must be outside the source checkout") + self.work.mkdir(parents=True, exist_ok=False) + self.evidence = self.work / "evidence" + self.evidence.mkdir() + self.env = dict(os.environ) + for name in list(self.env): + if name.startswith("GTEST_"): + self.env.pop(name) + self.env.update(GTEST_FILTER="*", GTEST_REPEAT="1") + self.steps = [] + self.results = {} + self.compiler = self.env.get("CC", "cc") + self.cxx = self.env.get("CXX", "c++") + self.prefixes = [path.resolve(strict=True) for path in args.dependency_prefix] + + def save(self, status): + (self.evidence / "summary.json").write_text(json.dumps({ + "status": status, "system": self.args.build_system, + "source": str(self.source), "steps": self.steps, "tests": self.results, + }, indent=2) + "\n") + + def step(self, name, command, cwd=None, timeout=900, env=None): + command = [str(part) for part in command] + log = self.evidence / (name + ".log") + directory = Path(cwd or self.source) + (self.evidence / (name + ".command")).write_text( + "cwd=" + str(directory) + "\n" + shlex.join(command) + "\n") + print("START", name, flush=True) + start = time.monotonic() + code = 1 + with log.open("w") as output: + process = subprocess.Popen(command, cwd=directory, env=env or self.env, + stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + stop_process(process) + code = 124 + print("Step exceeded timeout", file=output) + except BaseException: + stop_process(process) + raise + finally: + self.steps.append({"name": name, "exitcode": code, + "seconds": round(time.monotonic() - start, 3)}) + (self.evidence / (name + ".exitcode")).write_text(str(code) + "\n") + self.save("running") + print("END", name, "exit=" + str(code), flush=True) + if code != 0: + raise RuntimeError(name + " failed; see " + str(log)) + return log + + def download(self, name, version, digest): + archive = self.work / (name + ".tar.gz") + self.step("download-" + name, ["curl", "--fail", "--location", "--retry", "2", + "--connect-timeout", "20", "--max-time", "180", + "https://github.com/google/" + name + "/archive/refs/tags/v" + version + ".tar.gz", + "--output", archive], timeout=600) + return extract_archive(archive, digest, self.work / (name + "-source"), name + "-" + version) + + def prepare_dependencies(self): + if self.args.flatbuffers_prefix: + self.fb = self.args.flatbuffers_prefix.resolve(strict=True) + else: + source = self.download("flatbuffers", FLATBUFFERS_VERSION, FLATBUFFERS_SHA256) + self.fb = self.work / "dependencies" + build = self.work / "flatbuffers-build" + self.step("flatbuffers-configure", ["cmake", "-S", source, "-B", build, + "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", + "-DCMAKE_INSTALL_LIBDIR=lib", "-DCMAKE_INSTALL_PREFIX=" + str(self.fb), + "-DFLATBUFFERS_BUILD_TESTS=OFF", "-DFLATBUFFERS_BUILD_FLATC=ON", + "-DFLATBUFFERS_BUILD_FLATLIB=ON", "-DFLATBUFFERS_BUILD_SHAREDLIB=OFF", + "-DFLATBUFFERS_INSTALL=ON", "-DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF"]) + self.step("flatbuffers-build", ["cmake", "--build", build, "--parallel", self.args.jobs]) + self.step("flatbuffers-install", ["cmake", "--install", build]) + self.flatc = self.fb / "bin/flatc" + version = self.step("flatc-version", [self.flatc, "--version"], timeout=10).read_text().strip() + base = (self.fb / "include/flatbuffers/base.h").read_text() + numbers = [re.search(r"#define\s+FLATBUFFERS_VERSION_" + part + r"\s+(\d+)", base) + for part in ("MAJOR", "MINOR", "REVISION")] + if (version != "flatc version " + FLATBUFFERS_VERSION or + not all(numbers) or ".".join(match.group(1) for match in numbers) != FLATBUFFERS_VERSION): + raise ValueError("Use matching pinned FlatBuffers headers and compiler") + self.fb_library = self.fb / "lib/libflatbuffers.a" + if not self.fb_library.is_file(): + raise ValueError("The generator needs " + str(self.fb_library)) + self.gtest = (self.args.gtest_source.resolve(strict=True) if self.args.gtest_source else + self.download("googletest", GTEST_VERSION, GTEST_SHA256)) + self.prefixes.insert(0, self.fb) + if self.args.build_system == "make": + build = self.work / "gtest-build" + gtest_prefix = self.work / "gtest-prefix" + self.step("gtest-configure", ["cmake", "-S", self.gtest, "-B", build, + "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_INSTALL_LIBDIR=lib", + "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_INSTALL_PREFIX=" + str(gtest_prefix)]) + self.step("gtest-build", ["cmake", "--build", build, "--parallel", self.args.jobs]) + self.step("gtest-install", ["cmake", "--install", build]) + self.prefixes.insert(0, gtest_prefix) + self.env["PATH"] = os.pathsep.join(str(prefix / "bin") for prefix in self.prefixes) + os.pathsep + self.env.get("PATH", "") + self.step("protoc-version", [self.args.protoc or shutil.which("protoc", path=self.env["PATH"]), "--version"], timeout=10) + + def common_cmake(self): + options = ["-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", + "-DCMAKE_PREFIX_PATH=" + ";".join(str(prefix) for prefix in self.prefixes), + "-DCMAKE_C_COMPILER=" + self.compiler, "-DCMAKE_CXX_COMPILER=" + self.cxx] + for prefix in self.prefixes: + if (prefix / "include/openssl/ssl.h").is_file(): + options.append("-DOPENSSL_ROOT_DIR=" + str(prefix)) + break + if self.args.protoc: + options.append("-DProtobuf_PROTOC_EXECUTABLE=" + str(self.args.protoc.resolve(strict=True))) + return options + + def check_gtest(self, name, binary, ctest_build=None): + listing = self.step(name + "-list", [binary, "--gtest_list_tests"], timeout=30) + xml = self.evidence / (name + ".xml") + env = dict(self.env, GTEST_OUTPUT="xml:" + str(xml)) + command = (["ctest", "--test-dir", ctest_build, "--no-tests=error", "--output-on-failure", + "--timeout", "300", "-R", "^" + name + "$"] if ctest_build else [binary]) + self.step(name, command, timeout=330, env=env) + self.results[name] = verify_gtests(xml, listing, TESTS[name]) + self.save("running") + + def ctest(self, name, build, required): + xml = self.evidence / (name + ".xml") + self.step(name, ["ctest", "--test-dir", build, "--no-tests=error", "--output-on-failure", + "--timeout", "300", "--output-junit", xml], timeout=600) + self.results[name] = verify_ctest(xml, required) + self.save("running") + + def run_cmake(self): + build = self.work / "build" + runtime = self.work / "runtime" + self.step("configure", ["cmake", "-S", self.source, "-B", build] + self.common_cmake() + [ + "-DCMAKE_INSTALL_PREFIX=" + str(runtime), "-DINSTALL_GTEST=OFF", "-DWITH_FLATBUFFERS=ON", "-DBUILD_UNIT_TESTS=ON", "-DBUILD_BRPC_TOOLS=OFF", + "-DBUILD_SHARED_LIBS=ON", + "-DDOWNLOAD_GTEST=OFF", "-DBRPC_SYSTEM_GTEST_SOURCE_DIR=" + str(self.gtest), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATBUFFERS_FLATC_EXECUTABLE=" + str(self.flatc)]) + self.step("build", ["cmake", "--build", build, "--target", *TESTS, "brpc-shared", + "--parallel", self.args.jobs]) + self.step("install", ["cmake", "--install", build]) + check_enabled(runtime / "include/butil/config.h") + for name in TESTS: + self.check_gtest(name, build / "test" / name, build) + codegen = self.work / "codegen" + shared_library = "libbrpc.dylib" if sys.platform == "darwin" else "libbrpc.so" + self.step("codegen-configure", ["cmake", "-S", self.source / "tools/flatbuffers", "-B", codegen] + self.common_cmake() + [ + "-DBUILD_TESTING=ON", "-DBRPC_CODEGEN_BRPC_LIBRARY=" + str(build / "output/lib" / shared_library), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATBUFFERS_LIBRARY=" + str(self.fb_library), "-DFLATC_EXECUTABLE=" + str(self.flatc)]) + self.step("codegen-build", ["cmake", "--build", codegen, "--parallel", self.args.jobs]) + self.ctest("codegen", codegen, {"flatbuffers_codegen_acceptance", "flatbuffers_codegen_runtime"}) + example = self.work / "example" + self.step("example-configure", ["cmake", "-S", self.source / "example/benchmark_fb", "-B", example] + self.common_cmake() + [ + "-DBUILD_TESTING=ON", "-DBRPC_ROOT=" + str(runtime), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATC_EXECUTABLE=" + str(self.flatc), + "-DBRPC_FLATC_EXECUTABLE=" + str(codegen / "brpc_flatc")]) + self.step("example-build", ["cmake", "--build", example, "--parallel", self.args.jobs]) + self.ctest("example", example, {"benchmark_fb_smoke"}) + + def copy_source(self): + listing = self.step("source-files", ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"]) + destination = self.work / "source" + destination.mkdir() + count = 0 + for name in set(listing.read_bytes().split(b"\0")): + if not name: + continue + relative = Path(os.fsdecode(name)) + path = self.source / relative + if (relative.is_absolute() or ".." in relative.parts or path.is_symlink() or + self.source not in path.resolve().parents): + raise ValueError("Unsafe checkout path: " + str(relative)) + if not path.exists(): + continue + if not path.is_file(): + raise ValueError("Only regular source files are supported: " + str(relative)) + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + count += 1 + if not count: + raise ValueError("The source must be a nonempty Git checkout") + self.env["CCACHE_BASEDIR"] = str(destination) + return destination + + def run_make(self): + source = self.copy_source() + headers = [str(prefix / "include") for prefix in self.prefixes] + ["/usr/include"] + libs = [str(prefix / "lib") for prefix in self.prefixes] + ["/usr/lib", "/usr/lib64"] + self.step("configure", ["sh", "config_brpc.sh", "--with-flatbuffers", + "--headers=" + " ".join(headers), "--libs=" + " ".join(libs), + "--cc=" + self.compiler, "--cxx=" + self.cxx], cwd=source) + self.step("build", ["make", "-j" + str(self.args.jobs)], cwd=source) + check_enabled(source / "output/include/butil/config.h") + self.step("test-build", ["make", "-C", "test", "-j" + str(self.args.jobs), + "FLATC=" + str(self.flatc), *TESTS], cwd=source) + variable = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH" + self.env[variable] = os.pathsep.join([str(source / "test")] + + [str(prefix / "lib") for prefix in self.prefixes]) + for name in TESTS: + self.check_gtest(name, source / "test" / name) + + def run_bazel(self): + source = self.copy_source() + command = ["bazel", "--batch", "--output_user_root=" + str(self.work / "bazel-state"), + "test", "--define=BRPC_WITH_FLATBUFFERS=true", "--jobs=" + str(self.args.jobs), + "--local_test_jobs=1", "--test_timeout=300", "--cache_test_results=no", + "--runs_per_test=1", "--flaky_test_attempts=1", "--test_output=errors", + "--test_env=GTEST_FILTER=*", "--test_env=GTEST_REPEAT=1"] + try: + self.step("bazel-tests", command + ["//test:" + name for name in TESTS], cwd=source, timeout=1800) + finally: + for name in TESTS: + for filename in ("test.xml", "test.log"): + path = source / "bazel-testlogs/test" / name / filename + if path.is_file(): + shutil.copy2(path, self.evidence / (name + "." + filename)) + for name in TESTS: + listing = self.step(name + "-list", [source / "bazel-bin/test" / name, "--gtest_list_tests"], cwd=source, timeout=30) + self.results[name] = verify_gtests(self.evidence / (name + ".test.xml"), listing, TESTS[name]) + self.save("running") + + def run(self): + self.step("compiler-version", [self.cxx, "--version"], timeout=10) + self.step("cmake-version", ["cmake", "--version"], timeout=10) + if self.args.build_system == "bazel": + self.step("bazel-version", ["bazel", "--version"], timeout=30) + self.run_bazel() + else: + self.prepare_dependencies() + getattr(self, "run_" + self.args.build_system)() + self.save("passed") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-system", choices=("cmake", "make", "bazel"), required=True) + parser.add_argument("--source", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--jobs", type=int, default=2) + parser.add_argument("--flatbuffers-prefix", type=Path) + parser.add_argument("--gtest-source", type=Path) + parser.add_argument("--dependency-prefix", type=Path, action="append", default=[]) + parser.add_argument("--protoc", type=Path) + args = parser.parse_args() + if not 1 <= args.jobs <= 16: + parser.error("--jobs must be in 1..16") + runner = None + try: + runner = Runner(args) + runner.run() + except (Exception, KeyboardInterrupt) as error: + if runner: + runner.save("failed") + (runner.evidence / "failure.txt").write_text(str(error) + "\n") + print("FlatBuffers ON gate failed:", error, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt())) + sys.exit(main()) diff --git a/.github/scripts/test_flatbuffers_on.py b/.github/scripts/test_flatbuffers_on.py new file mode 100644 index 0000000000..4071f97645 --- /dev/null +++ b/.github/scripts/test_flatbuffers_on.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import base64 +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import signal +import select +import subprocess +import sys +import time +import tarfile +import tempfile +import unittest +from unittest import mock +import xml.etree.ElementTree as ET + +SCRIPT = Path(__file__).with_name("flatbuffers-on.py") +SPEC = importlib.util.spec_from_file_location("flatbuffers_on", SCRIPT) +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +class GateTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.listing = self.root / "list.txt" + self.listing.write_text("Suite.\n First\n Second\n") + self.xml = self.root / "tests.xml" + + def report(self, names=("First", "Second"), **attributes): + root = ET.Element("testsuites", tests=str(len(names)), failures="0", errors="0", disabled="0") + root.attrib.update(attributes) + suite = ET.SubElement(root, "testsuite", name="Suite") + for name in names: + ET.SubElement(suite, "testcase", classname="Suite", name=name, + status="run", result="completed") + ET.ElementTree(root).write(self.xml) + return root + + def save(self, root): + ET.ElementTree(root).write(self.xml) + + def test_successful_report_matches_discovery(self): + self.report() + self.assertEqual(2, gate.verify_gtests(self.xml, self.listing, 2)["executed"]) + + def test_parameterized_listing(self): + self.assertEqual({"Typed/0.Works/1"}, gate.list_gtests( + "Running main from gtest\nTyped/0. # TypeParam = int\n Works/1 # GetParam = 1\n")) + + def test_empty_or_disabled_listing_rejected(self): + for text in ("", "no tests", "DISABLED_Suite.\n Test\n", "Suite.\n DISABLED_Test\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + gate.list_gtests(text) + + def test_duplicate_listing_rejected(self): + with self.assertRaises(ValueError): + gate.list_gtests("Suite.\n First\n First\n") + + def test_empty_or_missing_xml_rejected(self): + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + self.report(()) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + + def test_filtered_and_unexpected_count_rejected(self): + self.report(("First",)) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + self.report() + # The expected count is exact: both fewer and more cases are rejected. + for count in (1, 3): + with self.subTest(count=count), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, count) + + def test_failure_error_skip_disabled_rejected(self): + for tag in ("failure", "error", "skipped"): + root = self.report() + ET.SubElement(next(root.iter("testcase")), tag) + self.save(root) + with self.subTest(tag=tag), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + for attribute in ("failures", "errors", "disabled", "skipped"): + self.report(**{attribute: "1"}) + with self.subTest(attribute=attribute), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_unexecuted_and_duplicate_cases_rejected(self): + root = self.report() + next(root.iter("testcase")).set("status", "notrun") + self.save(root) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + self.report(("First", "First")) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_incorrect_root_count_rejected(self): + self.report(tests="999") + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_ctest_requires_runtime_and_acceptance(self): + self.report(("acceptance", "runtime")) + self.assertEqual(2, gate.verify_ctest(self.xml, {"acceptance", "runtime"})["executed"]) + self.report(("acceptance",)) + with self.assertRaises(ValueError): + gate.verify_ctest(self.xml, {"acceptance", "runtime"}) + + def archive(self, name="pkg/file", link=False): + path = self.root / "archive.tar.gz" + with tarfile.open(path, "w:gz") as archive: + info = tarfile.TarInfo(name) + if link: + info.type = tarfile.SYMTYPE + info.linkname = "/outside" + archive.addfile(info) + else: + info.size = 4 + archive.addfile(info, io.BytesIO(b"data")) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + def test_pinned_archive_extracts(self): + path, digest = self.archive() + extracted = gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertEqual(b"data", (extracted / "file").read_bytes()) + + def test_archive_checksum_paths_and_links_rejected(self): + path, _ = self.archive() + with self.assertRaises(ValueError): + gate.extract_archive(path, "0" * 64, self.root / "extract", "pkg") + for name, link in (("../outside", False), ("pkg/../outside", False), + ("/absolute", False), ("wrong/file", False), ("pkg/link", True)): + path, digest = self.archive(name, link) + with self.subTest(name=name), self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def linked_archive(self, entries): + path = self.root / "linked.tar.gz" + with tarfile.open(path, "w:gz") as archive: + for name, target in entries: + info = tarfile.TarInfo(name) + if target is not None: + info.type = tarfile.SYMTYPE + info.linkname = target + archive.addfile(info) + else: + info.size = 4 + archive.addfile(info, io.BytesIO(b"data")) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + def test_internal_archive_symlinks_extract_after_data(self): + path, digest = self.linked_archive([ + ("pkg/java/src/test/java/Example", "../../../../tests/Example"), + ("pkg/ts/package.json", "../package.json"), + ("pkg/tests/Example/schema.fbs", None), ("pkg/package.json", None)]) + extracted = gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertTrue((extracted / "java/src/test/java/Example").is_symlink()) + self.assertEqual(b"data", (extracted / "java/src/test/java/Example/schema.fbs").read_bytes()) + self.assertEqual(b"data", (extracted / "ts/package.json").read_bytes()) + + def test_escaping_chained_and_ancestor_links_rejected(self): + cases = [ + [("pkg/link", "../../outside")], [("pkg", ".")], + [("pkg/link", "next"), ("pkg/next", "file"), ("pkg/file", None)], + [("pkg/dir", "target"), ("pkg/dir/file", None)], + [("pkg/a", "."), ("pkg/link", "nested/../a/../outside")], + ] + for entries in cases: + path, digest = self.linked_archive(entries) + with self.subTest(entries=entries), self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def test_duplicate_archive_members_rejected(self): + path, digest = self.linked_archive([("pkg/file", None), ("pkg/file", None)]) + with self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def runner(self): + source = self.root / "source" + source.mkdir() + return gate.Runner(argparse.Namespace(source=source, work=self.root / "work", + build_system="cmake", dependency_prefix=[])) + + def test_work_directory_is_exclusive(self): + runner = self.runner() + with self.assertRaises(FileExistsError): + gate.Runner(runner.args) + + def test_filter_environment_is_reset(self): + with mock.patch.dict(os.environ, {"GTEST_FILTER": "Wrong.*", "GTEST_SHARD_INDEX": "2"}): + runner = self.runner() + self.assertEqual("*", runner.env["GTEST_FILTER"]) + self.assertNotIn("GTEST_SHARD_INDEX", runner.env) + + def test_work_inside_checkout_rejected(self): + source = self.root / "checkout" + source.mkdir() + args = argparse.Namespace(source=source, work=source / "build", build_system="cmake", dependency_prefix=[]) + with self.assertRaises(ValueError): + gate.Runner(args) + + def test_source_directory_link_cannot_escape_checkout(self): + runner = self.runner() + outside = self.root / "outside" + outside.mkdir() + (outside / "file").write_text("must not be copied") + (runner.source / "link").symlink_to(outside, target_is_directory=True) + self.listing.write_bytes(b"link/file\0") + with mock.patch.object(runner, "step", return_value=self.listing): + with self.assertRaises(ValueError): + runner.copy_source() + self.assertFalse((runner.work / "source/link/file").exists()) + + def test_cmake_uses_explicit_openssl_prefix(self): + runner = self.runner() + runner.args.protoc = None + self.assertFalse(any(option.startswith("-DOPENSSL_ROOT_DIR=") + for option in runner.common_cmake())) + prefix = self.root / "openssl" + (prefix / "include/openssl").mkdir(parents=True) + (prefix / "include/openssl/ssl.h").write_text("header fixture") + runner.prefixes = [self.root / "unrelated", prefix] + self.assertIn("-DOPENSSL_ROOT_DIR=" + str(prefix), runner.common_cmake()) + + def test_make_builds_supplied_gtest_in_private_prefix(self): + runner = self.runner() + prefix = self.root / "provided-fb" + (prefix / "include/flatbuffers").mkdir(parents=True) + (prefix / "include/flatbuffers/base.h").write_text( + "#define FLATBUFFERS_VERSION_MAJOR 25\n#define FLATBUFFERS_VERSION_MINOR 2\n#define FLATBUFFERS_VERSION_REVISION 10\n") + (prefix / "lib").mkdir() + (prefix / "lib/libflatbuffers.a").touch() + source = self.root / "provided-gtest" + source.mkdir() + runner.args.build_system = "make" + runner.args.flatbuffers_prefix = prefix + runner.args.gtest_source = source + runner.args.jobs = 2 + runner.args.protoc = Path(sys.executable) + version = self.root / "version.log" + version.write_text("flatc version 25.2.10\n") + with mock.patch.object(runner, "step", return_value=version) as step: + with mock.patch.object(runner, "download") as download: + runner.prepare_dependencies() + download.assert_not_called() + configure = next(call.args[1] for call in step.call_args_list + if call.args[0] == "gtest-configure") + self.assertIn(source.resolve(), configure) + self.assertIn("-DCMAKE_INSTALL_PREFIX=" + str(runner.work / "gtest-prefix"), configure) + self.assertEqual(runner.work / "gtest-prefix", runner.prefixes[0]) + self.assertFalse((prefix / "include/gtest").exists()) + + def exercise_process_group(self, leader_exits): + ready = self.root / "child.ready" + child_code = ("import signal,time,pathlib; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "pathlib.Path(" + repr(str(ready)) + ").touch(); time.sleep(60)") + leader_code = ("import subprocess,sys,time,pathlib; " + "child=subprocess.Popen([sys.executable,'-c'," + repr(child_code) + "], " + "stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL); " + "print(child.pid,flush=True); " + + ("sys.exit(0)" if leader_exits else "time.sleep(60)")) + process = subprocess.Popen([sys.executable, "-c", leader_code], + stdout=subprocess.PIPE, text=True, start_new_session=True) + try: + readable, _, _ = select.select([process.stdout], [], [], 3) + self.assertTrue(readable, "group leader did not publish child PID") + child = int(process.stdout.readline()) + deadline = time.monotonic() + 3 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(ready.exists()) + if leader_exits: + process.wait(timeout=3) + gate.stop_process(process, grace=0.1) + deadline = time.monotonic() + 3 + while True: + result = subprocess.run(["ps", "-o", "stat=", "-p", str(child)], + stdout=subprocess.PIPE, text=True, check=False) + state = result.stdout.strip() + if not state or state.startswith("Z"): + break + self.assertLess(time.monotonic(), deadline, "owned child survived cleanup") + time.sleep(0.02) + finally: + gate.signal_group(process, signal.SIGKILL) + process.wait(timeout=3) + process.stdout.close() + + def test_cleanup_after_group_leader_exits(self): + self.exercise_process_group(True) + + def test_cleanup_kills_child_ignoring_term(self): + self.exercise_process_group(False) + + def test_timeout_is_a_recorded_failure(self): + runner = self.runner() + with self.assertRaises(RuntimeError): + runner.step("timeout", [sys.executable, "-c", "import time; time.sleep(60)"], timeout=0.05) + self.assertEqual(124, json.loads((runner.evidence / "summary.json").read_text())["steps"][0]["exitcode"]) + + def test_flatbuffers_pin_matches_both_bazel_definitions(self): + root = SCRIPT.resolve().parents[2] + for filename in ("MODULE.bazel", "WORKSPACE"): + text = (root / filename).read_text() + integrity = "sha256-" + base64.b64encode(bytes.fromhex(gate.FLATBUFFERS_SHA256)).decode() + self.assertTrue(gate.FLATBUFFERS_SHA256 in text or integrity in text, + filename + ": FlatBuffers checksum differs") + self.assertTrue("v" + gate.FLATBUFFERS_VERSION + ".tar.gz" in text, + filename + ": FlatBuffers version differs") + + def test_workflow_script_paths_and_characters(self): + root = SCRIPT.resolve().parents[2] + workflow = (root / ".github/workflows/flatbuffers-on.yml").read_text() + self.assertFalse(any(ord(char) < 32 and char not in "\n\r\t" for char in workflow)) + for filename in ("flatbuffers-on.py", "test_flatbuffers_on.py"): + self.assertIn(".github/scripts/" + filename, workflow) + self.assertTrue((SCRIPT.parent / filename).is_file()) + self.assertNotIn("pull_request_target", workflow) + self.assertNotIn("continue-on-error", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/flatbuffers-on.yml b/.github/workflows/flatbuffers-on.yml new file mode 100644 index 0000000000..3e754dd6b0 --- /dev/null +++ b/.github/workflows/flatbuffers-on.yml @@ -0,0 +1,130 @@ +name: FlatBuffers ON + +on: + push: + branches: [master] + paths: + - 'src/brpc/flatbuffers/**' + - 'src/brpc/policy/flatbuffers_protocol.*' + - 'src/brpc/channel.*' + - 'src/brpc/controller.*' + - 'src/brpc/server.*' + - 'src/brpc/global.cpp' + - 'src/brpc/options.proto' + - 'src/brpc/details/*_private_accessor.h' + - 'test/*flatbuffers*' + - 'test/CMakeLists.txt' + - 'test/Makefile' + - 'test/BUILD.bazel' + - 'tools/flatbuffers/**' + - 'example/benchmark_fb/**' + - 'CMakeLists.txt' + - 'src/**/CMakeLists.txt' + - 'Makefile' + - 'config_brpc.sh' + - 'BUILD.bazel' + - 'MODULE.bazel' + - 'WORKSPACE' + - '.github/scripts/*flatbuffers*' + - '.github/workflows/flatbuffers-on.yml' + pull_request: + paths: + - 'src/brpc/flatbuffers/**' + - 'src/brpc/policy/flatbuffers_protocol.*' + - 'src/brpc/channel.*' + - 'src/brpc/controller.*' + - 'src/brpc/server.*' + - 'src/brpc/global.cpp' + - 'src/brpc/options.proto' + - 'src/brpc/details/*_private_accessor.h' + - 'test/*flatbuffers*' + - 'test/CMakeLists.txt' + - 'test/Makefile' + - 'test/BUILD.bazel' + - 'tools/flatbuffers/**' + - 'example/benchmark_fb/**' + - 'CMakeLists.txt' + - 'src/**/CMakeLists.txt' + - 'Makefile' + - 'config_brpc.sh' + - 'BUILD.bazel' + - 'MODULE.bazel' + - 'WORKSPACE' + - '.github/scripts/*flatbuffers*' + - '.github/workflows/flatbuffers-on.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build-test: + name: FB ON (${{ matrix.os }}, ${{ matrix.system }}, ${{ matrix.compiler }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - {os: ubuntu-22.04, system: cmake, compiler: gcc, cc: gcc, cxx: g++, cache: ccache} + - {os: ubuntu-22.04, system: cmake, compiler: clang, cc: clang, cxx: clang++, cache: ccache} + - {os: ubuntu-22.04, system: make, compiler: gcc, cc: gcc, cxx: g++, cache: ccache} + - {os: ubuntu-22.04, system: make, compiler: clang, cc: clang, cxx: clang++, cache: ccache} + - {os: ubuntu-22.04, system: bazel, compiler: gcc, cc: gcc, cxx: g++, cache: bazel} + - {os: macos-latest, system: cmake, compiler: apple-clang, cc: clang, cxx: clang++, cache: ccache} + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + FB_WORK: ${{ runner.temp }}/brpc-fb-${{ matrix.system }}-${{ matrix.compiler }} + BUILD_SYSTEM: ${{ matrix.system }} + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Validate the gate's report checks + run: python3 .github/scripts/test_flatbuffers_on.py + + - name: Install focused Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build clang ccache pkg-config python3 curl ca-certificates git \ + libssl-dev libgflags-dev libprotobuf-dev libprotoc-dev protobuf-compiler \ + libleveldb-dev libgoogle-perftools-dev zlib1g-dev + + - name: Install focused macOS dependencies + if: runner.os == 'macOS' + run: brew install cmake ninja pkg-config openssl@3 gflags leveldb protobuf@29 abseil + + - uses: ./.github/actions/setup-build-cache + with: + kind: ${{ matrix.cache }} + cache-key: fb-on-${{ matrix.system }}-${{ matrix.compiler }} + + - name: Build and run the FB ON gates + shell: bash + run: | + args=(--build-system "$BUILD_SYSTEM" --work "$FB_WORK" --jobs 2) + if [[ "$RUNNER_OS" == macOS ]]; then + for package in protobuf@29 abseil openssl@3 gflags leveldb; do + args+=(--dependency-prefix "$(brew --prefix "$package")") + done + args+=(--protoc "$(brew --prefix protobuf@29)/bin/protoc") + fi + python3 .github/scripts/flatbuffers-on.py "${args[@]}" + + - name: Upload commands, logs and test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: flatbuffers-on-${{ matrix.os }}-${{ matrix.system }}-${{ matrix.compiler }} + path: ${{ env.FB_WORK }}/evidence/ + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 7a9ade11d1..7c646771b1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,10 @@ /test/output build/ -# Ignore hidden files +# Ignore hidden files, but keep GitHub workflows and helpers discoverable. .* +!/.github/ +/.github/scripts/__pycache__/ *.swp # Ignore auto-generated files diff --git a/BUILD.bazel b/BUILD.bazel index 5882a034da..67b2937b37 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -559,7 +559,10 @@ brpc_proto_library( visibility = ["//visibility:public"], ) -FLATBUFFERS_SRC_PATTERNS = ["src/brpc/flatbuffers/*.cpp"] +FLATBUFFERS_SRC_PATTERNS = [ + "src/brpc/flatbuffers/*.cpp", + "src/brpc/policy/flatbuffers_protocol.cpp", +] URMA_SRC_PATTERNS = [ "src/brpc/urma/*.cpp", diff --git a/CMakeLists.txt b/CMakeLists.txt index 0966595d13..48cce36cf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF) option(WITH_BORINGSSL "With BoringSSL" OFF) option(WITH_DEBUG_SYMBOLS "With debug symbols" ON) option(WITH_THRIFT "With thrift framed protocol supported" OFF) -option(WITH_FLATBUFFERS "With FlatBuffers message support (headers only)" OFF) +option(WITH_FLATBUFFERS "With FlatBuffers message and RPC support (headers only)" OFF) option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) option(WITH_SNAPPY "With snappy" OFF) option(WITH_RDMA "With RDMA" OFF) diff --git a/Makefile b/Makefile index 08df1f04e9..936299f363 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,9 @@ BRPC_SOURCES_ALL = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/*,$(SRCE ifeq ($(URMA_USE_MOCK),0) BRPC_SOURCES_ALL := $(filter-out src/brpc/urma/mock_urma.cpp,$(BRPC_SOURCES_ALL)) endif +ifneq ($(WITH_FLATBUFFERS),1) +BRPC_SOURCES_ALL := $(filter-out src/brpc/policy/flatbuffers_protocol.cpp,$(BRPC_SOURCES_ALL)) +endif BRPC_SOURCES = $(filter-out $(THRIFT_SOURCES) $(EXCLUDE_SOURCES), $(BRPC_SOURCES_ALL)) BRPC_PROTOS = $(filter %.proto,$(BRPC_SOURCES)) BRPC_CFAMILIES = $(filter-out %.proto %.pb.cc,$(BRPC_SOURCES)) diff --git a/README.md b/README.md index d65366fafb..3fa3ea7f08 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ You can use it to: * Read [overview](docs/en/overview.md) to know where bRPC can be used and its advantages. * Read [getting started](docs/en/getting_started.md) for building steps and play with [examples](https://github.com/apache/brpc/tree/master/example/). * Docs: + * [Enable FlatBuffers RPC and verify the example](docs/en/flatbuffers.md) * [Performance benchmark](docs/en/benchmark.md) * [bvar](docs/en/bvar.md) * [bvar_c++](docs/en/bvar_c++.md) diff --git a/README_cn.md b/README_cn.md index 2cc686bd85..b3a5946c0d 100644 --- a/README_cn.md +++ b/README_cn.md @@ -32,6 +32,7 @@ * 通过[概述](docs/cn/overview.md)了解哪里可以用bRPC及其优势。 * 阅读[编译步骤](docs/cn/getting_started.md)了解如何开始使用, 之后可以运行一下[示例程序](https://github.com/apache/brpc/tree/master/example/). * 文档: + * [启用 FlatBuffers RPC 并通过 example 验证](docs/cn/flatbuffers.md) * [性能测试](docs/cn/benchmark.md) * [bvar](docs/cn/bvar.md) * [bvar_c++](docs/cn/bvar_c++.md) diff --git a/docs/cn/flatbuffers.md b/docs/cn/flatbuffers.md index 5e58801594..fb2597b71b 100644 --- a/docs/cn/flatbuffers.md +++ b/docs/cn/flatbuffers.md @@ -1,52 +1,261 @@ -# FlatBuffers 消息 +# 启用 FlatBuffers 消息与 RPC [English version](../en/flatbuffers.md) -bRPC 支持基于 IOBuf 的 FlatBuffers 消息、构造器和服务描述符，默认不启用。 -消息构造方案源自 [apache/brpc#3196](https://github.com/apache/brpc/pull/3196)。 +bRPC 提供基于 IOBuf 的 FlatBuffers 消息、构造器、服务描述及 `fb_rpc` 协议。 +支持默认关闭，需先编译启用该功能的 bRPC，再让客户端和服务端使用同一套头文件、 +生成配置和运行库。完整示例位于 [example/benchmark_fb](../../example/benchmark_fb/README.md)。 -本组件不包含 `fb_rpc` 传输协议，也不改动 `brpc::Channel` 或 `brpc::Server`。 -服务代码生成和进程内分派测试不涉及网络 RPC，也不是性能测试。 +## 1. 启用选项与依赖 -## 构建 +| 构建方式 | 启用选项 | 可供独立 example 使用的运行库前缀 | +| --- | --- | --- | +| CMake | `-DWITH_FLATBUFFERS=ON` | `/output` | +| Make | `config_brpc.sh --with-flatbuffers` | `/output` | +| Bazel | build/test 命令均传 `--define=BRPC_WITH_FLATBUFFERS=true` | 原始 Bazel 输出不是 example 所需的安装前缀 | -运行库只需要 FlatBuffers 头文件，不链接其库；测试还需要版本匹配的 `flatc`。 -版本不一致时应重新生成头文件，不要删除或放宽上游生成代码中的版本断言。 +**不能只给应用增加 `-DBRPC_WITH_FLATBUFFERS=1`。** 该功能会改变 Channel、Controller、 +Server 的 ABI；ON 头文件与 OFF 运行库混用不受支持。`butil/config.h` 中的宏始终为 0 或 1， +应用应使用 `#if BRPC_WITH_FLATBUFFERS`，而不是 `#ifdef`。 -假设 GoogleTest 源码位于 `/usr/src/googletest`： +先按[入门指南](getting_started.md)准备 C++ 工具链、Protobuf 编译器及开发库、gflags、 +LevelDB、OpenSSL、zlib。FlatBuffers RPC **并不消除 Protobuf 依赖**。 -```sh -cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ - -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ - -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest -cmake --build build --target brpc_flatbuffers_unittest -j6 -ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure -``` +| 组件 | FlatBuffers 或测试依赖 | +| --- | --- | +| bRPC 消息及 RPC 运行库 | FlatBuffers 头文件，不链接 `libflatbuffers` | +| 官方 schema 代码生成 | 与头文件版本一致的 `flatc` | +| bRPC 服务代码生成 | `brpc_flatc`，其构建另需匹配的官方头文件和 `libflatbuffers` | +| example 烟测 | Python 3，不需要 GoogleTest | +| 库单元测试 | GoogleTest 及项目原有测试依赖 | -非标准安装可设置 `FLATBUFFERS_INCLUDE_DIR`、`FLATBUFFERS_FLATC_EXECUTABLE` -和 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。其他测试依赖与 bRPC 原有配置相同。 +本仓库的 Bazel 和 ON gate 固定使用 **FlatBuffers 25.2.10**。复现该流程时，应使用同版本的 +头文件、官方 `flatc` 和用于构建 `brpc_flatc` 的库。不要删除生成头文件中的版本断言。 -Make 使用 `config_brpc.sh --with-flatbuffers`，测试时可通过 -`FLATC=/path/to/flatc` 指定生成器。Bazel 使用 -`--define=BRPC_WITH_FLATBUFFERS=true`。 +三个容易混淆的参数： -运行库使用了 FlatBuffers builder 的内部结构，目前只支持 25.2.10，其他版本会在 -编译期报错。Bzlmod 和 WORKSPACE 使用同一个源码归档，并用校验和锁定内容。 -未采用 `bazel_dep`，是为了避免引入 gRPC、多语言工具以及与 bRPC 冲突的 BoringSSL -依赖。 +- 根项目 CMake 的库测试：`FLATBUFFERS_FLATC_EXECUTABLE` 指向官方 `flatc`。 +- 生成器验收及 example：`FLATC_EXECUTABLE` 指向官方 `flatc`。 +- example：`BRPC_FLATC_EXECUTABLE` 指向 bRPC 的 `brpc_flatc`，不能填成官方 `flatc`。 -公共头文件位于 `brpc/flatbuffers/`，命名空间为 `brpc::flatbuffers`。 -构造消息时包含 `message.h`；使用服务描述符和接口时包含 `service.h`。`butil/config.h` 中的 -`BRPC_WITH_FLATBUFFERS` 始终是 0 或 1，因此应使用 `#if` 判断。 +各阶段还必须使用兼容的同一套 Protobuf。当 CMake 检测到 `Protobuf_VERSION > 4.21` 时， +需要 C++17 及匹配的 Abseil 依赖；生成器验收和 example 需要 Protobuf 的 CMake config 包 +导出传递依赖。运行库与 example 最低要求 CMake 3.16；下文 CTest 命令使用 3.17+ 的 `--no-tests=error` +防止空测试误报成功。只有 3.16 时，可直接执行 `smoke.py`。以下按 Linux/macOS 的 +Unix Makefiles 或 Ninja 单配置生成器编写。 -旧版 flatc 2.0.x 可能生成未全限定名称。业务 schema 不要使用 `brpc` 命名空间，也不要依赖 include 顺序。 +运行库使用了 FlatBuffers builder 的内部结构，目前只支持 25.2.10，其他版本会在编译期报错。 +版本不一致时应重新生成头文件，不要删除或放宽上游生成代码中的版本断言。Bzlmod 和 WORKSPACE +使用同一个源码归档，并用校验和锁定内容；未采用 `bazel_dep`，是为了避免引入 gRPC、多语言工具 +以及与 bRPC 冲突的 BoringSSL 依赖。 如需使用其他 FlatBuffers 版本，需要修改运行库、生成器和 codegen 测试中的版本检查， 再用该版本完整运行 `tools/flatbuffers/verify.sh`。步骤见生成器 [README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version)， 其中以 23.5.26 为例，仅适用于 CMake 构建。 -## 消息构造和所有权 +## 2. 用 CMake 构建启用后的运行库 + +从 bRPC checkout 开始。按需提前设置有效的 `CC`、`CXX`。 +CMake 构建把生成文件放在 build 目录，example 的绑定代码与构建产物放在 `WORK` 中。 +独立 example 需要公共头文件和 `config.h`，build 目录里没有这些文件，所以要把运行库安装到一个前缀。 + +将下列变量替换为已有安装前缀，即包含 `include/`、`lib/` 或 `lib64/` 的目录；多个变量 +可以指向同一前缀。 + +```sh +REPO="$PWD" +DEPS=/absolute/path/to/dependency-prefix +FB=/absolute/path/to/flatbuffers-prefix +OPENSSL=/absolute/path/to/openssl-prefix +WORK="$(mktemp -d "${TMPDIR:-/tmp}/brpc-benchmark-fb.XXXXXX")" +JOBS=2 +printf 'WORK=%s\n' "$WORK" +``` + +若尚未安装 FlatBuffers，可先从匹配版本的官方源码构建。已有完整安装时跳过此段。 +`FB` 必须是可写的安装前缀，不是源码目录： + +```sh +FB_SOURCE=/absolute/path/to/flatbuffers-25.2.10-source +cmake -S "$FB_SOURCE" -B "$WORK/flatbuffers" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX="$FB" -DCMAKE_INSTALL_LIBDIR=lib \ + -DFLATBUFFERS_BUILD_TESTS=OFF -DFLATBUFFERS_BUILD_FLATC=ON \ + -DFLATBUFFERS_BUILD_FLATLIB=ON -DFLATBUFFERS_BUILD_SHAREDLIB=OFF \ + -DFLATBUFFERS_INSTALL=ON -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF +cmake --build "$WORK/flatbuffers" --parallel "$JOBS" +cmake --install "$WORK/flatbuffers" +"$FB/bin/flatc" --version +``` + +构建静态运行库，不启用库单元测试，因此这一阶段不需要 GoogleTest： + +```sh +cmake -S "$REPO" -B "$WORK/runtime" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DCMAKE_INSTALL_PREFIX="$WORK/prefix" \ + -DWITH_FLATBUFFERS=ON -DBUILD_SHARED_LIBS=OFF \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DBUILD_UNIT_TESTS=OFF -DDOWNLOAD_GTEST=OFF -DBUILD_BRPC_TOOLS=OFF +cmake --build "$WORK/runtime" --parallel "$JOBS" +cmake --install "$WORK/runtime" +grep -E '^#define BRPC_WITH_FLATBUFFERS +1$' "$WORK/prefix/include/butil/config.h" +``` + +最后一条检查必须输出 `#define BRPC_WITH_FLATBUFFERS 1`。公共接口位于 +`brpc/flatbuffers/message.h` 和 `brpc/flatbuffers/service.h`。 +上面的 policy 参数是兼容 CMake 4 下某些旧依赖策略的可选设置，不是启用 FlatBuffers 的 +开关，也不能替代编译器和依赖版本要求。 + +## 3. 通过 example 完成端到端验证 + +该示例默认做**有界功能验证**，另有可选的计时模式，用于对比 `fb_rpc` 和等价的 `baidu_std` 服务。[echo.fbs](../../example/benchmark_fb/echo.fbs) +声明 `BenchmarkService.Echo`，显式 wire method ID 为 7。 +[server.cpp](../../example/benchmark_fb/server.cpp) 使用 `AddFlatBuffersService` 注册服务； +[client.cpp](../../example/benchmark_fb/client.cpp) 使用生成的 `BenchmarkService::Stub` +及 `fb_rpc` channel。 + +### 构建生成器和示例 + +在前一节同一个 shell 中继续，沿用 `REPO`、`DEPS`、`FB`、`OPENSSL`、`WORK`、`JOBS`： + +```sh +cmake -S "$REPO/tools/flatbuffers" -B "$WORK/codegen" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH="$FB;$DEPS" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" +cmake --build "$WORK/codegen" --parallel "$JOBS" + +cmake -S "$REPO/example/benchmark_fb" -B "$WORK/example" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=ON \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DBRPC_ROOT="$WORK/prefix" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DFLATC_EXECUTABLE="$FB/bin/flatc" \ + -DBRPC_FLATC_EXECUTABLE="$WORK/codegen/brpc_flatc" +cmake --build "$WORK/example" --parallel "$JOBS" +(cd "$WORK/example" && ctest -V --no-tests=error --output-on-failure -R '^benchmark_fb_smoke$') +``` + +构建时会执行两个生成器，文件位于 `WORK/example/generated/`： + +- 官方 `flatc`：`echo_generated.h`，定义 table 类型。 +- `brpc_flatc`：`echo.brpc.fb.h`、`echo.brpc.fb.cpp`，定义服务及 stub。 + +不要提交这些生成文件，也不要在更换版本后沿用旧头文件。生成器若选错库，可在其 CMake +配置中显式传入 `FLATBUFFERS_LIBRARY`。导入 schema、稳定方法 ID 等规则见 +[生成器说明](../../tools/flatbuffers/README.md)。 + +### 烟测的成功标准 + +必须看到名为 `benchmark_fb_smoke` 的 **1 项 CTest 通过**、命令退出码为 0，并输出： + +```text +benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown +``` + +这是 15 次 RPC（13 次正常响应、2 次预期 schema 拒绝），不是 15 项 CTest。`No tests were found` +不算通过。烟测验证的内容、期限，以及不经 CTest 直接运行的方法见 +[example README](../../example/benchmark_fb/README.md#smoke-test)。 + +### 手工分别运行 server/client + +手工运行客户端的命令、预期结果和有界参数见 +[example README](../../example/benchmark_fb/README.md#run-separately)。 + +### 复用已有运行库或改为动态链接 + +已有兼容的 FB ON 运行库时，无需重编：将 example 配置中的 `BRPC_ROOT` 替换为其安装前缀 +（CMake）或 `output/`（Make），`BRPC_FLATC_EXECUTABLE` 指向匹配的生成器，其他步骤不变。 +example 会读取该前缀真实的 `include/butil/config.h`。 + +默认链接 `libbrpc.a`。使用 `LINK_SO=ON` 前，先用 `BUILD_SHARED_LIBS=ON` 构建 `brpc-shared` +并再次执行 `cmake --install`，再用 `-DLINK_SO=ON` 重新配置并构建 example；`LINK_SO` 本身 +不会构建运行库。运行库若增加可选依赖，可通过 `BRPC_EXTRA_LIBRARIES` 补充链接库。 + +## 4. Make、Bazel 和完整回归 + +### Make + +在独立可写 checkout 中配置，沿用前述依赖变量： + +```sh +sh config_brpc.sh --with-flatbuffers \ + --headers="$FB/include $DEPS/include $OPENSSL/include" \ + --libs="$DEPS/lib $OPENSSL/lib" --cc="${CC:-cc}" --cxx="${CXX:-c++}" +make -j"$JOBS" +``` + +按平台替换为实际 `lib64` 或库目录。产出的 `output/` 可以替换 example 命令中的 `$WORK/prefix`，作为 `BRPC_ROOT`。 +Make 单测通过 `FLATC="$FB/bin/flatc"` 选择官方生成器，但还需要已安装的 GoogleTest 库和 +gperftools，不是只提供 GoogleTest 源码即可；执行时也需能够加载 `test/libbrpc.dbg.*`。 +下方 ON runner 会构建 GoogleTest、设置测试库路径并校验报告；gperftools 等系统测试依赖 +仍需预先安装。 + +### Bazel + +```sh +bazel build --define=BRPC_WITH_FLATBUFFERS=true //:brpc +bazel test --define=BRPC_WITH_FLATBUFFERS=true --cache_test_results=no \ + //test:brpc_flatbuffers_unittest //test:brpc_flatbuffers_protocol_unittest +``` + +这些目标验证库，不会自动执行独立 example。不要将原始 `bazel-bin` 当成 `BRPC_ROOT`； +example 需要 CMake 安装前缀或 Make `output/` 的 include/lib 布局。 + +### 库单测与 ON gate + +单独配置库单测时，设置 `BUILD_UNIT_TESTS=ON`、匹配的 `FLATBUFFERS_FLATC_EXECUTABLE`； +若 `DOWNLOAD_GTEST=OFF`，还需指定 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。 +构建并运行 `brpc_flatbuffers_unittest`、`brpc_flatbuffers_protocol_unittest`。 +仅打开库的 FlatBuffers 选项，不会自动执行测试。 + +要一次验证两组库单测、两项 codegen 验收及 example，可使用 +[ON runner](../../.github/scripts/flatbuffers-on.py)。完整 CMake 门禁需要 **CMake/CTest 3.21+** +以生成 JUnit XML 报告（`--output-junit`）。**`--work` 目录必须尚不存在**： + +```sh +python3 "$REPO/.github/scripts/flatbuffers-on.py" \ + --build-system cmake --source "$REPO" --work "$WORK/on-gate" --jobs "$JOBS" \ + --flatbuffers-prefix "$FB" \ + --dependency-prefix "$DEPS" --dependency-prefix "$OPENSSL" +``` + +该门禁要求 FlatBuffers 25.2.10。不传 `--flatbuffers-prefix` 时会下载、校验并构建该版本。 +GoogleTest 默认下载并校验固定的 1.14.0，也可用 `--gtest-source` 复用源码。 +平台常规依赖仍需预先安装。Make/Bazel 模式验证两组库测试，只有 CMake 模式额外构建 +生成器和 example。 + +检查 `WORK/on-gate/evidence/summary.json`：`status` 应为 `passed`，应包含所有预期测试， +执行数量非零，失败和跳过均为零。各步骤命令、日志及 XML 即使失败也会保留； +空测试、过滤后少跑或跳过不会被当作成功。 + +[CI 工作流](../../.github/workflows/flatbuffers-on.yml)覆盖 Linux CMake/Make 的 GCC、Clang， +Linux Bazel GCC，以及 macOS CMake。本地 gate 通过，不代表 GitHub 托管矩阵或另一套依赖 +组合已经通过。 + +## 5. 常见问题 + +| 现象 | 检查与处理 | +| --- | --- | +| `BRPC_ROOT is not FlatBuffers-enabled` 或缺少 FB 符号 | 用正确选项重编运行库和使用方，检查安装前缀里的 `include/butil/config.h`；不要强制宏或混用 ON 头文件与 OFF 库。 | +| 找不到 `flatbuffers/idl.h`、`libflatbuffers` | 生成器需要完整官方开发安装，单有运行时头文件不够；明确指定 include 和 library 路径。 | +| 头文件与 `flatc` 版本不匹配 | 检查 `flatbuffers/base.h` 和 `flatc --version`，统一版本，在新目录重新生成代码，不删除版本断言。 | +| Protobuf/Abseil 缺头文件或链接符号 | 统一 Protobuf，提供其 config 包和 Abseil 前缀，避免系统与私有安装混用。 | +| macOS 找不到 OpenSSL | 设置实际 `OPENSSL_ROOT_DIR` 和 `CMAKE_PREFIX_PATH`，保持编译器、SDK、架构一致。 | +| 无法写 `src/butil/config.h.tmp` | 只有 `config_brpc.sh`（Make）会往源码目录写这个文件，所以源码必须可写；用独立副本隔离。CMake 把 `config.h` 放在 build 目录。 | +| CMake 4 报旧依赖策略不兼容 | 对相关依赖使用适当的 policy compatibility 设置或升级依赖；不能借此降低 C++ 和依赖版本要求。 | +| `No tests were found` | 检查 example 的 build 目录、`BUILD_TESTING=ON` 和已构建的程序，或直接运行 `smoke.py`。 | +| `LINK_SO=ON` 找不到库或动态加载失败 | 先用 `BUILD_SHARED_LIBS=ON` 构建 `brpc-shared`，再检查共享库及传递依赖的运行时搜索路径。 | +| 连接拒绝或超时 | 等待 readiness，用当前端口，确保同一主机/容器且服务端未超过生命周期；不能计作 schema 拒绝成功。 | + +## 6. 消息构造和所有权 先用上游 `flatc --cpp` 生成 `*_generated.h`，再将 `brpc::flatbuffers::MessageBuilder` 传给生成的 `Create...` 函数。 @@ -86,7 +295,7 @@ schema 校验前复制到新存储，因此大小限制必须放在解析之前 `GetMutableRoot()`。合法消息中的可选 string/vector 仍可能为 null。 Framing 检查失败不会修改原 Message。 -## Service ID 和代码生成 +## 7. Service ID 和代码生成 `BrpcDescriptorTable` 保存 namespace、service 名称、按空白分隔的方法名和显式 方法 ID。ID 必须是唯一的非负 int32。手写 descriptor 可以传空 ID 列表，此时按 @@ -114,3 +323,28 @@ rpc_service BenchmarkService { 生成的 dispatch 会验证请求，拒绝未知、不属于当前 service 或尚未实现的方法。 失败时，它会执行非空 completion callback；成功分派后，completion 由业务实现负责， 并且必须恰好执行一次。 + +## 8. 网络 RPC + +库和使用方必须用相同的 `BRPC_WITH_FLATBUFFERS` 设置构建；启用该功能会改变 Channel、Controller、Server 的 ABI。用 `server.AddFlatBuffersService(&service, SERVER_DOESNT_OWN_SERVICE)` 注册生成的服务；将 `ChannelOptions::protocol` 设为 `"fb_rpc"`，并把这个 `brpc::Channel` 传给生成的 stub。stub 调用 `Channel::FBCallMethod`；`Controller::flatbuffers_method()` 返回带类型的 descriptor，protobuf 的 `Controller::method()` 仍为空。手工提供的 descriptor 必须活过整个调用，包括重试和 backup request。 + +服务注册独立于 protobuf 的 AddService 和 ListServices。Add/Remove/Clear 要求服务端处于已停止（READY）状态，不能是正在 STOPPING；管理操作必须由外部串行化。只有注册成功后所有权才转移。`RemoveFlatBuffersService` 会删除自有服务，Stop/Join 则保留注册以便重启。`GetFlatBuffersServiceCount()` 返回这个独立注册表的数量。重复的 service ID（包括 hash 冲突）以及与 protobuf 方法全名冲突的 FlatBuffers 方法会被拒绝。按字符串调用的 `MaxConcurrencyOf` 支持 FlatBuffers 方法，服务端整体并发和默认方法并发限制同样适用。 + +传输层支持同步/异步调用、重试、backup request、single/pooled/short 连接以及请求/响应附件。每次发送都会分配一个独立的小 header 并共享 payload 存储，不会改写 const 请求的 metadata 前缀，因此重试和并发调用可以安全共享同一个不可变请求。应用仍须保证每个 response 和 Controller 活到调用完成。 + +Framing 只检查长度，不检查应用 schema。生成的服务会在分派前验证请求，手写服务也必须这样做。调用方使用 root accessor 前必须验证收到的响应 schema，RPC 成功不能替代 `response.Verify()`。 + +不支持鉴权、压缩、校验和与 streaming，遇到时会明确拒绝而不是悄悄忽略。FlatBuffers 服务不能通过仅限内置服务的内部端口访问。本传输层不提供 SelectiveChannel/ParallelChannel、HTTP/JSON 映射和 RPC-dump 回放，也不传输 log ID、用户字段和分布式追踪元数据。 + +完整的客户端/服务端示例见[第 3 节](#3-通过-example-完成端到端验证)；示例演示这些 API，不做性能结论。 + +### FRPC 帧格式与兼容性 + +一帧由 `[12 字节 header][metadata][message][attachment]` 组成。header 包含 `FRPC`、大端 uint32 的 body 大小和大端 uint32 的 metadata 大小；body 大小不含 12 字节 header，但包含后面三段。metadata 前缀明确使用小端，与原先在小端机器上的 FRPC 实验一致，且不依赖 packed 结构体： + +* 请求：uint32 service ID、int32 method ID、int32 message 大小、int32 attachment 大小、uint64 correlation ID（共 24 字节）。 +* 响应：int32 错误码、int32 message 大小、int32 attachment 大小、uint64 correlation ID（共 20 字节）。 + +读取方要求完整的已知前缀，然后根据 header 中的 metadata 大小跳过未知的尾部。以后新增的可选字段只能追加，不要重排、改变大小或改变已有字段的含义。前缀过短、大小为负、payload 长度不一致或 body 过大都会被拒绝。错误响应带非零错误码且没有 payload。Method ID 是稀疏的 wire ID，不是数组下标：只有保留显式 ID 时，增加、删除或重排声明才不会改变已有方法的 ID。这并不意味着旧的按序号编号的 schema 能兼容，已删除的 ID 也绝不能复用。 + +magic 是 `FRPC`，不是 `BRPC`。该格式不保证与更早未发布、使用不同 magic、service hash 或本机大端 metadata 的版本兼容。库的全局 Protocol hook 签名保持不变，现有协议回调实现无需修改。自定义的数字协议 ID 不得与新分配的内置 ID 重叠，`PROTOCOL_FLATBUFFERS_RPC` 使用 ID 30。 diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md index 50748900ad..cc767fa721 100644 --- a/docs/en/flatbuffers.md +++ b/docs/en/flatbuffers.md @@ -1,50 +1,57 @@ -# FlatBuffers messages +# FlatBuffers messages and RPC [中文版](../cn/flatbuffers.md) -bRPC supports IOBuf-backed FlatBuffers messages, builders, and service -descriptors, disabled by default. Message construction is based on -[apache/brpc#3196](https://github.com/apache/brpc/pull/3196). +bRPC provides optional IOBuf-backed FlatBuffers messages, builders, service +descriptors, and the `fb_rpc` transport. The implementation builds on +[apache/brpc#3196](https://github.com/apache/brpc/pull/3196) and +[apache/brpc#3197](https://github.com/apache/brpc/pull/3197), while preserving +Protocol's existing protobuf callback signatures. -This component does not include `fb_rpc` transport or change `brpc::Channel` -or `brpc::Server`. Service generation and in-process dispatch tests do not -exercise network RPC or measure performance. +## Enable FlatBuffers -## Build +FlatBuffers is **OFF by default**. Enable it when building bRPC, then build the +client/server against that same library and generated configuration header. +Adding `-DBRPC_WITH_FLATBUFFERS=1` to an application is not a substitute: the +feature changes the Channel, Controller and Server ABI. -The runtime needs only FlatBuffers headers; tests also need a matching `flatc`. -Regenerate incompatible headers rather than removing or weakening upstream -version assertions. +| Build system | Enable option | Exported runtime prefix | +| --- | --- | --- | +| CMake | `-DWITH_FLATBUFFERS=ON` | `cmake --install` prefix | +| Make | `config_brpc.sh --with-flatbuffers` | `/output` | +| Bazel | `--define=BRPC_WITH_FLATBUFFERS=true` on build/test commands | Bazel outputs, not an install prefix for the standalone example | -With GoogleTest sources at `/usr/src/googletest`: +### Dependencies and versions -```sh -cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ - -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ - -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest -cmake --build build --target brpc_flatbuffers_unittest -j6 -ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure -``` - -For nonstandard installations, set `FLATBUFFERS_INCLUDE_DIR`, -`FLATBUFFERS_FLATC_EXECUTABLE`, and `BRPC_SYSTEM_GTEST_SOURCE_DIR` as needed. -The usual bRPC test dependencies still apply. +Prepare the normal bRPC dependencies described in [Getting started](getting_started.md): +a C++ toolchain, Protobuf compiler/development libraries, gflags, LevelDB, +OpenSSL and zlib. FlatBuffers RPC does **not** remove the Protobuf dependency. -For Make, use `config_brpc.sh --with-flatbuffers` and set `FLATC=/path/to/flatc` -when testing. For Bazel, use `--define=BRPC_WITH_FLATBUFFERS=true`. +| Component | Additional requirement | +| --- | --- | +| bRPC message/RPC runtime | FlatBuffers headers; no `libflatbuffers` linkage | +| Official schema generation | `flatc` matching the runtime headers | +| bRPC service generation | `brpc_flatc`, built with matching official headers and `libflatbuffers` | +| Example smoke | Python 3; no GoogleTest requirement | +| Library unit tests | GoogleTest and the project's test dependencies | -The runtime depends on FlatBuffers builder internals and supports only 25.2.10; -other versions fail at compile time. Bzlmod and WORKSPACE use the same -checksum-pinned archive. Using this instead of `bazel_dep` avoids gRPC, -other language tools, and conflicting BoringSSL dependencies. +The repository's Bazel/ON gate pins FlatBuffers **25.2.10**. Use a complete, +matching installation to reproduce it; never remove the generated header's +version assertions. CMake runtime tests use `FLATBUFFERS_FLATC_EXECUTABLE`, +whereas generator acceptance and the example use `FLATC_EXECUTABLE`. +`BRPC_FLATC_EXECUTABLE` always means the bRPC generator, not official `flatc`. -Public headers are in `brpc/flatbuffers/`, with namespace `brpc::flatbuffers`. -Include `message.h` for message construction and `service.h` for service -descriptors and interfaces. `BRPC_WITH_FLATBUFFERS` in `butil/config.h` is -0 or 1; test it with `#if`. +Use the same Protobuf installation across all builds. With `Protobuf_VERSION > +4.21`, C++17 and Abseil are required, and the generator and example builds need +Protobuf's CMake config package. CMake 3.16+ is enough to build; the CTest +commands below use 3.17+ for `--no-tests=error` (with 3.16, run `smoke.py` +directly). Use a single-configuration generator such as Unix Makefiles or Ninja. -Older flatc 2.0.x may emit unqualified names. Keep business schemas outside -the `brpc` namespace and do not rely on include order. +The runtime depends on FlatBuffers builder internals and supports only +25.2.10; other versions fail at compile time. Regenerate incompatible headers +rather than removing or weakening upstream version assertions. Bzlmod and +WORKSPACE use the same checksum-pinned archive; this avoids the gRPC, other +language tools and conflicting BoringSSL dependencies that `bazel_dep` would add. To use another FlatBuffers version, update the version checks in the runtime, the generator and the codegen tests, then run `tools/flatbuffers/verify.sh` @@ -52,6 +59,215 @@ against that version. See the generator [README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version) for the steps, using 23.5.26 as an example. They apply to CMake builds only. +### CMake runtime build + +Run from a checkout, with a working compiler selected through `CC`/`CXX` if +necessary. The CMake build keeps generated files in its build directory, and the +example bindings and binaries stay in `WORK`. Install the runtime into a prefix: +the build tree alone does not contain the public headers or `config.h` that the +standalone example needs. + +Replace the prefixes below with your installations (`include/`, `lib/` or +`lib64/`); one prefix may serve several dependencies. + +```sh +REPO="$PWD" +DEPS=/absolute/path/to/dependency-prefix +FB=/absolute/path/to/flatbuffers-prefix +OPENSSL=/absolute/path/to/openssl-prefix +WORK="$(mktemp -d "${TMPDIR:-/tmp}/brpc-benchmark-fb.XXXXXX")" +JOBS=2 +printf 'WORK=%s\n' "$WORK" +``` + +If FlatBuffers is not installed, build a matching official source checkout first +(skip this block when `FB` already contains the required headers, compiler and +library). `FB` must be a writable installation prefix, not the source directory: + +```sh +FB_SOURCE=/absolute/path/to/flatbuffers-25.2.10-source +cmake -S "$FB_SOURCE" -B "$WORK/flatbuffers" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX="$FB" -DCMAKE_INSTALL_LIBDIR=lib \ + -DFLATBUFFERS_BUILD_TESTS=OFF -DFLATBUFFERS_BUILD_FLATC=ON \ + -DFLATBUFFERS_BUILD_FLATLIB=ON -DFLATBUFFERS_BUILD_SHAREDLIB=OFF \ + -DFLATBUFFERS_INSTALL=ON -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF +cmake --build "$WORK/flatbuffers" --parallel "$JOBS" +cmake --install "$WORK/flatbuffers" +"$FB/bin/flatc" --version +``` + +Build the runtime without the unit-test dependencies: + +```sh +cmake -S "$REPO" -B "$WORK/runtime" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DCMAKE_INSTALL_PREFIX="$WORK/prefix" \ + -DWITH_FLATBUFFERS=ON -DBUILD_SHARED_LIBS=OFF \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DBUILD_UNIT_TESTS=OFF -DDOWNLOAD_GTEST=OFF -DBUILD_BRPC_TOOLS=OFF +cmake --build "$WORK/runtime" --parallel "$JOBS" +cmake --install "$WORK/runtime" +grep -E '^#define BRPC_WITH_FLATBUFFERS +1$' "$WORK/prefix/include/butil/config.h" +``` + +The final check must print `#define BRPC_WITH_FLATBUFFERS 1`. Public headers live +in `brpc/flatbuffers/`: `message.h` for messages/builders and `service.h` for +services/descriptors. The configuration macro is always 0 or 1; use `#if`, not +`#ifdef`. The policy argument above is an optional compatibility setting for +older dependencies under CMake 4, not a FlatBuffers switch or a replacement for +required dependency versions. + +### Make and Bazel alternatives + +In a separate writable checkout, using the same dependency prefixes: + +```sh +sh config_brpc.sh --with-flatbuffers \ + --headers="$FB/include $DEPS/include $OPENSSL/include" \ + --libs="$DEPS/lib $OPENSSL/lib" --cc="${CC:-cc}" --cxx="${CXX:-c++}" +make -j"$JOBS" +``` + +Use `lib64` or the platform's library directory where appropriate. The resulting +`output/` can replace `WORK/prefix` in the example command below. Make +unit tests take `FLATC="$FB/bin/flatc"` and require installed GoogleTest libraries +and gperftools, not just GoogleTest sources; their `test/libbrpc.dbg.*` must also +be loadable. The ON runner below builds GoogleTest, sets the test library path, +and validates reports; system test dependencies such as gperftools must already +be installed. + +Bazel supplies its pinned dependencies; pass the feature flag to both commands: + +```sh +bazel build --define=BRPC_WITH_FLATBUFFERS=true //:brpc +bazel test --define=BRPC_WITH_FLATBUFFERS=true --cache_test_results=no \ + //test:brpc_flatbuffers_unittest //test:brpc_flatbuffers_protocol_unittest +``` + +These Bazel targets verify the library, not the standalone example. Do not use a +raw `bazel-bin` directory as `BRPC_ROOT`; the example requires the include/lib +layout of a CMake install prefix or a Make `output/` directory. + +## Verify with the client/server example + +[example/benchmark_fb](../../example/benchmark_fb/README.md) runs a **bounded +functional check** by default and has an opt-in timed mode that compares `fb_rpc` +with an equivalent `baidu_std` service. Its [schema](../../example/benchmark_fb/echo.fbs) +uses `BenchmarkService.Echo` with explicit wire ID 7. The +[server](../../example/benchmark_fb/server.cpp) calls `AddFlatBuffersService`; +the [client](../../example/benchmark_fb/client.cpp) uses the generated +`BenchmarkService::Stub` and `fb_rpc` channel. + +### Generate, build and smoke-test + +Continue in the same shell with the variables and runtime from above. An +existing compatible FB ON runtime may be used instead by changing `BRPC_ROOT`. +Build the bRPC generator, then the example: + +```sh +cmake -S "$REPO/tools/flatbuffers" -B "$WORK/codegen" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH="$FB;$DEPS" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" +cmake --build "$WORK/codegen" --parallel "$JOBS" + +cmake -S "$REPO/example/benchmark_fb" -B "$WORK/example" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=ON \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DBRPC_ROOT="$WORK/prefix" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DFLATC_EXECUTABLE="$FB/bin/flatc" \ + -DBRPC_FLATC_EXECUTABLE="$WORK/codegen/brpc_flatc" +cmake --build "$WORK/example" --parallel "$JOBS" +(cd "$WORK/example" && ctest -V --no-tests=error --output-on-failure -R '^benchmark_fb_smoke$') +``` + +The example build runs **both** generators. `flatc` produces `echo_generated.h`; +`brpc_flatc` produces `echo.brpc.fb.h/.cpp`, all under `WORK/example/generated/`. +Do not commit them or reuse stale generated headers after changing versions. +If library discovery is ambiguous, pass `FLATBUFFERS_LIBRARY` explicitly when +configuring `brpc_flatc`. See the [generator guide](../../tools/flatbuffers/README.md) +for included schemas, explicit IDs and generator acceptance tests. + +Success means **one named CTest passed**, exit status 0, and this output: + +```text +benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown +``` + +That is 15 RPCs, not 15 CTests. `No tests were found` is not a pass. What the +smoke test checks, its deadlines and how to run it without CTest are in the +[example README](../../example/benchmark_fb/README.md#smoke-test). + +The example links `libbrpc.a` by default. For `LINK_SO=ON`, build and install the +runtime with `BUILD_SHARED_LIBS=ON` and the `brpc-shared` target first; `LINK_SO` +alone cannot create a shared runtime. + +### Run the programs separately + +The [example README](../../example/benchmark_fb/README.md#run-separately) lists the +manual client commands, their expected results and the bounded client/server +options. + +## Additional tests and the ON gate + +For library unit tests, configure with `BUILD_UNIT_TESTS=ON`, +`FLATBUFFERS_FLATC_EXECUTABLE` pointing to the matching official compiler, and +`BRPC_SYSTEM_GTEST_SOURCE_DIR` pointing to GoogleTest sources when +`DOWNLOAD_GTEST=OFF`. Build and run `brpc_flatbuffers_unittest` and +`brpc_flatbuffers_protocol_unittest`; enabling the library alone does not run them. + +For the complete CMake path (both library suites, two codegen tests, and the +example smoke), the shared [ON runner](../../.github/scripts/flatbuffers-on.py) +can use the prefixes above. This full CMake gate requires **CMake/CTest 3.21+** +for JUnit XML output (`--output-junit`). **Its work directory must not already +exist**: + +```sh +python3 "$REPO/.github/scripts/flatbuffers-on.py" \ + --build-system cmake --source "$REPO" --work "$WORK/on-gate" --jobs "$JOBS" \ + --flatbuffers-prefix "$FB" \ + --dependency-prefix "$DEPS" --dependency-prefix "$OPENSSL" +``` + +This gate requires FlatBuffers 25.2.10. Omit `--flatbuffers-prefix` to let it +download/checksum/build that version; GoogleTest defaults to a checksum-pinned +1.14.0 download, or can be supplied with `--gtest-source`. Normal platform +dependencies must already be installed. Make/Bazel modes validate the two +library suites; only CMake mode also builds the generator and example. Inspect +`WORK/on-gate/evidence/summary.json` for `status: passed`, all required test names, +nonzero executed counts and zero failed/skipped cases. Commands, logs and XML +reports are retained even on failure. Empty/filtered/skipped runs are rejected. + +The [workflow](../../.github/workflows/flatbuffers-on.yml) runs Linux CMake/Make +with GCC and Clang, Linux Bazel with GCC, and macOS CMake. A local gate pass does +not imply the hosted GitHub matrix or a different dependency combination passed. + +## Troubleshooting + +| Symptom | Check / action | +| --- | --- | +| `BRPC_ROOT is not FlatBuffers-enabled` or missing FB symbols | Rebuild bRPC with the enable option, then rebuild all consumers. Check the installed prefix's `include/butil/config.h`; do not force the macro or mix an ON header with an OFF library. | +| Missing `flatbuffers/idl.h` or `libflatbuffers` | The service generator needs the full official development installation, not only the runtime headers. Set `FLATBUFFERS_INCLUDE_DIR` and, if needed, `FLATBUFFERS_LIBRARY`. | +| Header/compiler version mismatch | Check `flatc --version`, select matching headers/compiler/library, and regenerate bindings in a fresh build directory. Do not delete upstream version checks. | +| Missing Protobuf/Abseil headers or link symbols | Use one compatible Protobuf installation throughout and expose its CMake config and Abseil prefixes. Do not mix system and private headers/libraries. | +| OpenSSL not found on macOS | Set `OPENSSL_ROOT_DIR` and include the actual installed prefix in `CMAKE_PREFIX_PATH`; check compiler/SDK/architecture consistency. | +| Cannot write `src/butil/config.h.tmp` | Only `config_brpc.sh` (Make) writes this file into the checkout, so it needs a writable source tree. Use a private copy; CMake keeps `config.h` in its build directory. | +| Old dependency policy error with CMake 4 | Try the appropriate `CMAKE_POLICY_VERSION_MINIMUM` compatibility setting for that dependency, or update it; this does not change the required C++ or dependency versions. | +| `No tests were found` | Use the example build directory, configure `BUILD_TESTING=ON`, build the executables, then run the named test or `smoke.py` directly. | +| `LINK_SO=ON` cannot find a library or the loader fails | Build `brpc-shared` with runtime `BUILD_SHARED_LIBS=ON` first. Check the shared library's dependencies and runtime search paths. | +| Connection refused/timeout | Wait for the readiness line, use its current port on the same host/container, and check the server's finite lifetime. These are not successful schema rejections. | + +Flatc 2.0.x emits unqualified `flatbuffers::` names. Keep business schemas outside +the `brpc` namespace (for example `myapp.rpc`); do not depend on include order to +avoid shadowing. + ## Message construction and ownership Generate `*_generated.h` with upstream `flatc --cpp`. Pass a @@ -127,11 +343,82 @@ rpc_service BenchmarkService { success. Generated accessors use thread-safe function-local static initialization. -The generator in `tools/flatbuffers/` uses the upstream parser and links -`libflatbuffers` separately. See its [README](../../tools/flatbuffers/README.md) -for commands and limitations. +The companion generator in `tools/flatbuffers/` uses the upstream parser to +produce service bindings. It is independently built; only that optional tool +needs `libflatbuffers`. See its [README](../../tools/flatbuffers/README.md) for +commands and limitations. Generated dispatch verifies requests, rejects +unknown/foreign methods, and runs non-null completion callbacks on failure, +including unimplemented methods. Successful implementations own completion and +must run their callback exactly once. + +## Network RPC + +Build both the library and its users with the same `BRPC_WITH_FLATBUFFERS` +setting; enabling the feature changes the Channel, Controller and Server ABI. +Register generated services with `server.AddFlatBuffersService(&service, +SERVER_DOESNT_OWN_SERVICE)`. Use `ChannelOptions::protocol = "fb_rpc"` and pass +that `brpc::Channel` to a generated stub. The stub calls `Channel::FBCallMethod`; +`Controller::flatbuffers_method()` returns the typed descriptor, while the +protobuf `Controller::method()` remains null. Manually supplied descriptors +must outlive their calls, including retries and backup requests. + +Service registration is separate from protobuf's AddService and ListServices +APIs. Add/remove/clear require a stopped (READY) server, not a server still +STOPPING; management operations must be externally serialized. Ownership +transfers only after successful registration. `RemoveFlatBuffersService` +deletes an owned service, while Stop/Join keep registrations for restart. +`GetFlatBuffersServiceCount()` reports this separate registry. Duplicate service +IDs (including hash collisions) and conflicting full protobuf/FlatBuffers method +names are rejected. String-based `MaxConcurrencyOf` supports FlatBuffers +methods, as do server-wide and default method concurrency limits. + +The transport supports synchronous/asynchronous calls, retries, backup requests, +single/pooled/short connections and request/response attachments. It allocates a +small independent header for each send and shares payload storage; it never +rewrites a const request's metadata prefix. Thus retries and concurrent calls +can safely share an immutable request. The application must still keep each +response and Controller alive until completion. + +Framing validates lengths, not application schemas. Generated services verify +requests before dispatch. Handwritten services must do the same. Callers must +verify received response schemas before using root accessors; an RPC succeeding +does not replace `response.Verify()`. + +Authentication, compression, checksums and streaming are not supported and are +rejected rather than silently ignored. FlatBuffers services cannot be accessed +through the internal, builtin-only port. SelectiveChannel/ParallelChannel, +HTTP/JSON mapping and RPC-dump replay are not provided by this transport. +Log IDs, user fields and distributed-tracing metadata are not transmitted. + +For a complete generated client/server, follow +[Verify with the client/server example](#verify-with-the-clientserver-example) +above. The example demonstrates these APIs without making a performance claim. + +### FRPC framing and compatibility + +A frame is `[12-byte header][metadata][message][attachment]`. The header contains +`FRPC`, a big-endian uint32 body size, and a big-endian uint32 metadata size; body +size excludes the 12-byte header but includes all three following sections. +The metadata prefix is explicitly little-endian, matching the original FRPC +experiment on little-endian machines without relying on packed structs: + +* Request: uint32 service ID, int32 method ID, int32 message size, int32 attachment + size, uint64 correlation ID (24 bytes). +* Response: int32 error code, int32 message size, int32 attachment size, uint64 + correlation ID (20 bytes). + +Readers require the whole known prefix, then skip unknown trailing metadata +using the advertised metadata size. Future optional fields must be appended; +do not reorder, resize or repurpose existing fields. A shorter prefix, negative +size, inconsistent payload length or excessive body is rejected. Error replies +have a nonzero error code and no payload. Method IDs are sparse wire IDs, never +array positions: adding, deleting or reordering declarations preserves surviving +IDs only when explicit IDs are retained. This does not make legacy ordinal-ID +schemas compatible, and removed IDs must never be reused. -Generated dispatch validates requests and rejects unknown, foreign, or -unimplemented methods. On failure it runs a non-null completion callback. After -successful dispatch, the application owns completion and must run it exactly -once. +The magic is `FRPC`, not `BRPC`. This format does not promise compatibility with +older unpublished variants that used different magic, service hashes or native +big-endian metadata. The library's global Protocol hook signatures remain +unchanged, so existing protocol callback implementations need no adaptation. +Private numeric protocol IDs must not overlap newly assigned builtin IDs; +`PROTOCOL_FLATBUFFERS_RPC` uses ID 30. diff --git a/example/benchmark_fb/CMakeLists.txt b/example/benchmark_fb/CMakeLists.txt new file mode 100644 index 0000000000..780b108583 --- /dev/null +++ b/example/benchmark_fb/CMakeLists.txt @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.16...3.28) +project(benchmark_fb LANGUAGES CXX) + +set(BRPC_ROOT "" CACHE PATH "FlatBuffers-enabled brpc output or install prefix") +set(BRPC_EXTRA_LIBRARIES "" CACHE STRING "Extra dependencies of this libbrpc, as a CMake list") +option(LINK_SO "Link the shared brpc library instead of libbrpc.a" OFF) +if(NOT BRPC_ROOT) + message(FATAL_ERROR "Set BRPC_ROOT to an existing WITH_FLATBUFFERS=ON output/install prefix") +endif() +get_filename_component(BRPC_ROOT "${BRPC_ROOT}" ABSOLUTE) +set(BRPC_INCLUDE_DIR "${BRPC_ROOT}/include") +if(NOT EXISTS "${BRPC_INCLUDE_DIR}/brpc/channel.h" OR + NOT EXISTS "${BRPC_INCLUDE_DIR}/butil/config.h") + message(FATAL_ERROR "BRPC_ROOT must contain include/brpc/channel.h and include/butil/config.h") +endif() +file(STRINGS "${BRPC_INCLUDE_DIR}/butil/config.h" _fb_enabled + REGEX "^#[ \t]*define[ \t]+BRPC_WITH_FLATBUFFERS[ \t]+1([ \t]|$)") +if(NOT _fb_enabled) + message(FATAL_ERROR + "BRPC_ROOT is not FlatBuffers-enabled. Rebuild brpc with WITH_FLATBUFFERS=ON; do not force BRPC_WITH_FLATBUFFERS with -D.") +endif() +# Re-resolve when BRPC_ROOT or LINK_SO changes; never select another installed brpc. +unset(_BRPC_LIBRARY CACHE) +if(LINK_SO) + set(_brpc_names "${CMAKE_SHARED_LIBRARY_PREFIX}brpc${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(_brpc_names libbrpc.a) +endif() +find_library(_BRPC_LIBRARY NAMES ${_brpc_names} + PATHS "${BRPC_ROOT}/lib" "${BRPC_ROOT}/lib64" NO_DEFAULT_PATH) +if(NOT _BRPC_LIBRARY) + message(FATAL_ERROR "The requested brpc library was not found under BRPC_ROOT/lib or lib64") +endif() + +find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) +find_program(FLATC_EXECUTABLE NAMES flatc) +find_program(BRPC_FLATC_EXECUTABLE NAMES brpc_flatc HINTS "${BRPC_ROOT}/bin") +if(NOT FLATBUFFERS_INCLUDE_DIR OR NOT FLATC_EXECUTABLE OR NOT BRPC_FLATC_EXECUTABLE) + message(FATAL_ERROR "Set FLATBUFFERS_INCLUDE_DIR, FLATC_EXECUTABLE and BRPC_FLATC_EXECUTABLE") +endif() +if(NOT EXISTS "${FLATBUFFERS_INCLUDE_DIR}/flatbuffers/base.h") + message(FATAL_ERROR "FLATBUFFERS_INCLUDE_DIR must contain flatbuffers/base.h") +endif() +foreach(_part MAJOR MINOR REVISION) + file(STRINGS "${FLATBUFFERS_INCLUDE_DIR}/flatbuffers/base.h" _version_line + REGEX "^#define[ \t]+FLATBUFFERS_VERSION_${_part}[ \t]+[0-9]+") + if(NOT _version_line MATCHES "FLATBUFFERS_VERSION_${_part}[ \t]+([0-9]+)") + message(FATAL_ERROR "Cannot read FlatBuffers ${_part} version from base.h") + endif() + set(_fb_${_part} "${CMAKE_MATCH_1}") +endforeach() +set(_headers_version "${_fb_MAJOR}.${_fb_MINOR}.${_fb_REVISION}") +execute_process(COMMAND "${FLATC_EXECUTABLE}" --version + RESULT_VARIABLE _flatc_result OUTPUT_VARIABLE _flatc_version + ERROR_VARIABLE _flatc_error OUTPUT_STRIP_TRAILING_WHITESPACE TIMEOUT 5) +if(NOT _flatc_result STREQUAL "0" OR + NOT _flatc_version MATCHES "flatc version ([0-9]+\\.[0-9]+\\.[0-9]+)") + message(FATAL_ERROR "Cannot execute official flatc --version: ${_flatc_error}") +endif() +set(_compiler_version "${CMAKE_MATCH_1}") +if(NOT _compiler_version VERSION_EQUAL _headers_version) + message(FATAL_ERROR + "FlatBuffers headers ${_headers_version} do not match flatc ${_compiler_version}") +endif() +execute_process(COMMAND "${BRPC_FLATC_EXECUTABLE}" --help + RESULT_VARIABLE _brpc_flatc_result OUTPUT_QUIET ERROR_VARIABLE _brpc_flatc_error TIMEOUT 5) +if(NOT _brpc_flatc_result STREQUAL "0") + message(FATAL_ERROR "Cannot execute BRPC_FLATC_EXECUTABLE: ${_brpc_flatc_error}") +endif() + +find_package(Threads REQUIRED) +find_package(Protobuf CONFIG QUIET) +set(_protobuf_config ${Protobuf_FOUND}) +if(NOT Protobuf_FOUND) + find_package(Protobuf REQUIRED) +endif() +find_package(OpenSSL REQUIRED) +find_package(ZLIB REQUIRED) +find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h) +find_library(GFLAGS_LIBRARY NAMES gflags) +find_library(LEVELDB_LIBRARY NAMES leveldb) +if(NOT GFLAGS_INCLUDE_DIR OR NOT GFLAGS_LIBRARY OR NOT LEVELDB_LIBRARY) + message(FATAL_ERROR "Install gflags and leveldb, or set CMAKE_PREFIX_PATH to the runtime dependency prefix") +endif() +set(_cxx_standard 14) +if(Protobuf_VERSION VERSION_GREATER 4.21) + set(_cxx_standard 17) + if(NOT _protobuf_config) + message(FATAL_ERROR "Protobuf 5+ needs its CMake config package to export transitive Abseil dependencies") + endif() +endif() + +add_library(benchmark_fb_dependencies INTERFACE) +# Keep the actual runtime configuration ahead of dependency include directories. +target_include_directories(benchmark_fb_dependencies INTERFACE + "${BRPC_INCLUDE_DIR}" "${FLATBUFFERS_INCLUDE_DIR}" "${GFLAGS_INCLUDE_DIR}") +target_compile_features(benchmark_fb_dependencies INTERFACE cxx_std_${_cxx_standard}) +target_link_libraries(benchmark_fb_dependencies INTERFACE + "${_BRPC_LIBRARY}" protobuf::libprotobuf "${GFLAGS_LIBRARY}" "${LEVELDB_LIBRARY}" + OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB Threads::Threads ${CMAKE_DL_LIBS} + ${BRPC_EXTRA_LIBRARIES}) +file(STRINGS "${BRPC_INCLUDE_DIR}/butil/config.h" _glog_enabled + REGEX "^#[ \t]*define[ \t]+BRPC_WITH_GLOG[ \t]+1([ \t]|$)") +if(_glog_enabled) + find_package(glog CONFIG REQUIRED) + target_link_libraries(benchmark_fb_dependencies INTERFACE glog::glog) +endif() +if(APPLE) + target_link_libraries(benchmark_fb_dependencies INTERFACE + "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreData" + "-framework CoreText" "-framework Security" "-framework Foundation") + target_link_options(benchmark_fb_dependencies INTERFACE + "LINKER:-U,_MallocExtension_ReleaseFreeMemory" "LINKER:-U,_ProfilerStart" + "LINKER:-U,_ProfilerStop" "LINKER:-U,__Z13GetStackTracePPvii" + "LINKER:-U,_mallctl" "LINKER:-U,_malloc_stats_print") +endif() + +set(_generated "${CMAKE_CURRENT_BINARY_DIR}/generated") +set(_schema "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs") +set(_generated_files "${_generated}/echo_generated.h" + "${_generated}/echo.brpc.fb.h" "${_generated}/echo.brpc.fb.cpp") +add_custom_command(OUTPUT ${_generated_files} + COMMAND "${CMAKE_COMMAND}" -E make_directory "${_generated}" + COMMAND "${FLATC_EXECUTABLE}" --cpp -o "${_generated}" "${_schema}" + COMMAND "${BRPC_FLATC_EXECUTABLE}" -o "${_generated}" "${_schema}" + DEPENDS "${_schema}" "${FLATC_EXECUTABLE}" "${BRPC_FLATC_EXECUTABLE}" + VERBATIM) +add_custom_target(benchmark_fb_codegen DEPENDS ${_generated_files}) +# Protobuf control schema used for the baidu_std comparison. +if(TARGET protobuf::protoc) + set(_protoc "$") +elseif(Protobuf_PROTOC_EXECUTABLE) + set(_protoc "${Protobuf_PROTOC_EXECUTABLE}") +else() + message(FATAL_ERROR "Set Protobuf_PROTOC_EXECUTABLE to the protoc matching libprotobuf") +endif() +add_custom_command(OUTPUT "${_generated}/echo_pb.pb.h" "${_generated}/echo_pb.pb.cc" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${_generated}" + COMMAND "${_protoc}" "--cpp_out=${_generated}" -I "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/echo_pb.proto" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/echo_pb.proto" + VERBATIM) +add_custom_target(benchmark_fb_pb_codegen DEPENDS + "${_generated}/echo_pb.pb.h" "${_generated}/echo_pb.pb.cc") +add_dependencies(benchmark_fb_codegen benchmark_fb_pb_codegen) +add_library(benchmark_fb_schema STATIC "${_generated}/echo.brpc.fb.cpp" + "${_generated}/echo_pb.pb.cc") +add_dependencies(benchmark_fb_schema benchmark_fb_codegen) +target_include_directories(benchmark_fb_schema PUBLIC "${_generated}") +target_link_libraries(benchmark_fb_schema PUBLIC benchmark_fb_dependencies) +foreach(_program client server) + add_executable(benchmark_fb_${_program} "${_program}.cpp") + add_dependencies(benchmark_fb_${_program} benchmark_fb_codegen) + target_link_libraries(benchmark_fb_${_program} PRIVATE benchmark_fb_schema) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(benchmark_fb_${_program} PRIVATE -Wall -Wextra) + endif() +endforeach() + +include(CTest) +if(BUILD_TESTING) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + add_test(NAME benchmark_fb_smoke + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/smoke.py" + --server "$" + --client "$") + set_tests_properties(benchmark_fb_smoke PROPERTIES TIMEOUT 50 LABELS "flatbuffers;smoke") +endif() +message(STATUS "benchmark_fb: brpc=${_BRPC_LIBRARY}, FlatBuffers=${_headers_version}") diff --git a/example/benchmark_fb/README.md b/example/benchmark_fb/README.md new file mode 100644 index 0000000000..a3713ecbc3 --- /dev/null +++ b/example/benchmark_fb/README.md @@ -0,0 +1,141 @@ +# FlatBuffers RPC example + +[English guide](../../docs/en/flatbuffers.md) | +[中文指南](../../docs/cn/flatbuffers.md) + +Independent client and server programs using the `fb_rpc` protocol, a generated +FlatBuffers service, and `brpc::flatbuffers::Message`. By default the client +runs a **bounded functional check**. A timed mode compares `fb_rpc` with an +equivalent protobuf service over `baidu_std` (see +[Timed comparison](#timed-comparison)). + +The schema declares `BenchmarkService.Echo` with stable wire method ID **7**. +The generated stub calls `Channel::FBCallMethod`, and the server registers the +generated service with `Server::AddFlatBuffersService`. The same server also +registers `benchmark_fb_pb.BenchmarkService` from `echo_pb.proto`, a protobuf +service with the same fields, and serves both `fb_rpc` and `baidu_std`. + +## Build + +Dependencies, the FlatBuffers-enabled runtime install, the generator and the +example build are described in the guide's +[verification section](../../docs/en/flatbuffers.md#verify-with-the-clientserver-example). +`BRPC_ROOT` must be an install prefix (or a Make `output/` directory) of a bRPC +built with FlatBuffers ON; the example reads its `include/butil/config.h` and +rejects an OFF build. Both `flatc` and `brpc_flatc` write generated code only to +the build directory; do not commit `*_generated.h` or `*.brpc.fb.*`. + +To link the shared library, build and install the runtime with +`-DBUILD_SHARED_LIBS=ON` and the `brpc-shared` target, then configure the +example with `-DLINK_SO=ON`. Extra runtime dependencies can be passed with +`-DBRPC_EXTRA_LIBRARIES='library1;library2'`. + +## Run separately + +By default the server and client accept only IPv4 loopback endpoints. In the +build terminal: + +```sh +"$WORK/example/benchmark_fb_server" --listen_addr=127.0.0.1:0 --duration_s=300 +``` + +The server prints `BRPC_FB_READY 127.0.0.1:` after `Server::Start` +succeeds. Use that port in another terminal (shell variables are not shared, so +set `WORK` again): + +```sh +WORK=/absolute/path/printed/by/the/build +SERVER=127.0.0.1:PORT_FROM_READY +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=16 --thread_num=2 --request_size=8193 --attachment_size=257 +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 --request_size=0 --omit_message=true +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 --corrupt_request=true +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 +``` + +All four commands exit 0. The first prints +`{"completed":16,"successes":16,"expected_rejections":0,"failures":0}`. The +corrupt-request run prints `{"completed":2,"successes":0,"expected_rejections":2,"failures":0}`, +and the last one checks recovery on the same server (`completed=2`, +`successes=2`). + +The client verifies the response schema before reading it, then compares the +request ID, every string byte, optional-string presence, and the attachment. +`--corrupt_request=true` damages the FlatBuffers root offset but keeps the FRPC +framing valid. It succeeds only if every request gets the generated service's +schema rejection (error code -1); timeouts and connection errors are failures. +`failures` is a 0/1 flag, not a count, and no throughput is reported. + +### Bounded controls + +| Client option | Default | Allowed | +| --- | --- | --- | +| `server` | required | `127.0.0.1:`, any IPv4 with `allow_remote` | +| `protocol` | `fb_rpc` | `fb_rpc`, `baidu_std` (protobuf control service) | +| `request_count` | 16 | 1..1000 total, not per thread | +| `thread_num` | 2 | 1..16 synchronous callers (1..1024 when timed) | +| `request_size` | 64 | 0..1048576 string bytes | +| `attachment_size` | 0 | 0..1048576 bytes | +| `timeout_ms` | 1500 | 1..5000 per RPC | +| `deadline_ms` | 30000 | 1..120000 for the client run | +| `connection_type` | `single` | `single`, `pooled`, `short` | +| `omit_message` | false | Preserve absent rather than empty string | +| `corrupt_request` | false | Require schema rejection (`fb_rpc`, count mode only) | +| `duration_s` | 0 | 0 for count mode, 1..3600 for timed mode | +| `warmup_s` | 0 | 0..600 unmeasured seconds before a timed run | +| `allow_remote` | false | Accept a non-loopback `server` | +| `use_bthread` | false | Run callers as bthreads, like `multi_threaded_echo_c++ --use_bthread` | + +The server defaults to `127.0.0.1:0`, `duration_s=60` (1..86400) and +`max_concurrency=32` (1..4096), and stops earlier on SIGINT/SIGTERM. Use +`--allow_remote` with a non-loopback `listen_addr`. Builtin +services are disabled. FRPC has no authentication, compression, checksums or +streaming, so this is not a production or public-network configuration. + +## Timed comparison + +With `--duration_s` the client keeps one synchronous RPC in flight per caller +thread, verifies every reply, and prints one JSON line with QPS, payload +throughput, latency percentiles and the client's CPU cores. Run each protocol +with identical sizes and concurrency against the same server: + +```sh +"$WORK/example/benchmark_fb_server" --listen_addr=SERVER_IP:8901 --allow_remote \ + --duration_s=3600 --max_concurrency=4096 +for p in fb_rpc baidu_std; do + "$WORK/example/benchmark_fb_client" --server=SERVER_IP:8901 --allow_remote \ + --protocol=$p --duration_s=30 --warmup_s=3 --thread_num=64 --request_size=8192 +done +``` + +Both services echo the request string and attachment; the FlatBuffers path +builds and verifies a FlatBuffers response, while the protobuf path parses and +serializes the equivalent message. The results are only comparable when +binaries, hosts, concurrency, sizes and duration are the same. + +## Smoke test + +CTest runs the standard-library-only `smoke.py`, which starts its own server on +an ephemeral loopback port and checks: + +1. finite RPCs with binary attachments, empty and absent strings, and + single/pooled/short connections (13 verified replies); +2. two malformed-schema requests are rejected; +3. normal requests still work on the same server afterwards; +4. the server it started is stopped and reaped. + +The orchestration deadline is 35 seconds and CTest's timeout is 50 seconds. +Needing the SIGKILL fallback fails the test. The smoke test can also run +without CTest: + +```sh +python3 "$REPO/example/benchmark_fb/smoke.py" \ + --server "$WORK/example/benchmark_fb_server" \ + --client "$WORK/example/benchmark_fb_client" +``` + +For common failures see the guide's +[troubleshooting table](../../docs/en/flatbuffers.md#troubleshooting). diff --git a/example/benchmark_fb/client.cpp b/example/benchmark_fb/client.cpp new file mode 100644 index 0000000000..d28d2268b3 --- /dev/null +++ b/example/benchmark_fb/client.cpp @@ -0,0 +1,454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "butil/config.h" +#include "butil/endpoint.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "bthread/bthread.h" +#include "echo.brpc.fb.h" +#include "echo_pb.pb.h" + +#if !BRPC_WITH_FLATBUFFERS +#error "benchmark_fb requires a FlatBuffers-enabled brpc library" +#endif + +DEFINE_string(server, "", "IPv4 endpoint printed by benchmark_fb_server; " + "a non-loopback address also needs --allow_remote"); +DEFINE_bool(allow_remote, false, "Allow a non-loopback --server for cross-host benchmarks"); +DEFINE_string(protocol, "fb_rpc", "fb_rpc, or baidu_std for the protobuf control service"); +DEFINE_string(connection_type, "single", "single, pooled, or short"); +DEFINE_int32(request_count, 16, "Count mode: total requests across all threads, 1..1000"); +DEFINE_int32(duration_s, 0, "0 selects count mode; 1..3600 runs a timed benchmark"); +DEFINE_int32(warmup_s, 0, "Timed mode: unmeasured warm-up seconds, 0..600"); +DEFINE_int32(thread_num, 2, "Concurrent synchronous callers: 1..16, or 1..1024 when timed"); +DEFINE_int32(request_size, 64, "Bytes in the optional request string, 0..1048576"); +DEFINE_int32(attachment_size, 0, "Bytes in the echoed attachment, 0..1048576"); +DEFINE_int32(timeout_ms, 1500, "Per-RPC timeout, 1..5000 milliseconds"); +DEFINE_int32(deadline_ms, 30000, "Count mode: overall client deadline, 1..120000 milliseconds"); +DEFINE_bool(omit_message, false, "Omit the optional string rather than sending an empty string"); +DEFINE_bool(use_bthread, false, "Run callers as bthreads instead of pthreads"); +DEFINE_bool(corrupt_request, false, "fb_rpc only: send an invalid root offset and " + "require schema rejection"); + +namespace { +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using Clock = std::chrono::steady_clock; + +// Result of one RPC: kSuccess, kRejected (expected schema rejection) or kFailed. +enum CallResult { kSuccess, kRejected, kFailed }; +using CallFn = std::function; + +std::string MakeBytes(size_t size, unsigned multiplier) { + std::string bytes(size, '\0'); + for (size_t i = 0; i < size; ++i) { + bytes[i] = static_cast((i * multiplier) % 251); + } + return bytes; +} + +Message MakeRequest(uint64_t request_id, const std::string& text) { + MessageBuilder builder(text.size() + 256); + const auto message = FLAGS_omit_message ? + ::flatbuffers::Offset<::flatbuffers::String>() : + builder.CreateString(text.data(), text.size()); + builder.Finish(benchmark_fb::CreateRequest(builder, request_id, message)); + Message request = builder.ReleaseMessage(); + if (FLAGS_corrupt_request) { + // Damage only the schema, not FRPC framing, so the server's generated + // dispatcher must reject the request before invoking the service. + memset(request.mutable_data(), 0xff, sizeof(::flatbuffers::uoffset_t)); + } + return request; +} + +bool CheckResponse(const Message& response, uint64_t request_id, + const std::string& text, const std::string& attachment, + const brpc::Controller& cntl) { + if (!response.Verify()) { + return false; + } + const auto* output = response.GetRoot(); + if (output->request_id() != request_id || + output->attachment_size() != attachment.size() || + cntl.response_attachment().to_string() != attachment) { + return false; + } + if (FLAGS_omit_message) { + return output->message() == nullptr; + } + return output->message() != nullptr && output->message()->size() == text.size() && + memcmp(output->message()->data(), text.data(), text.size()) == 0; +} + +bool CheckPbResponse(const benchmark_fb_pb::Response& response, uint64_t request_id, + const std::string& text, const std::string& attachment, + const brpc::Controller& cntl) { + if (response.request_id() != request_id || + response.attachment_size() != attachment.size() || + cntl.response_attachment().to_string() != attachment) { + return false; + } + if (FLAGS_omit_message) { + return !response.has_message(); + } + return response.has_message() && response.message() == text; +} + +bool IsLoopback(const std::string& server) { + return server.compare(0, 10, "127.0.0.1:") == 0; +} + +bool ValidFlags(butil::EndPoint* endpoint) { + const bool timed = FLAGS_duration_s > 0; + return (IsLoopback(FLAGS_server) || FLAGS_allow_remote) && + butil::str2endpoint(FLAGS_server.c_str(), endpoint) == 0 && endpoint->port > 0 && + (FLAGS_protocol == "fb_rpc" || FLAGS_protocol == "baidu_std") && + !(FLAGS_corrupt_request && (FLAGS_protocol != "fb_rpc" || timed)) && + (FLAGS_connection_type == "single" || FLAGS_connection_type == "pooled" || + FLAGS_connection_type == "short") && + FLAGS_request_count >= 1 && FLAGS_request_count <= 1000 && + FLAGS_duration_s >= 0 && FLAGS_duration_s <= 3600 && + FLAGS_warmup_s >= 0 && FLAGS_warmup_s <= 600 && (timed || FLAGS_warmup_s == 0) && + FLAGS_thread_num >= 1 && FLAGS_thread_num <= (timed ? 1024 : 16) && + FLAGS_request_size >= 0 && FLAGS_request_size <= 1024 * 1024 && + FLAGS_attachment_size >= 0 && FLAGS_attachment_size <= 1024 * 1024 && + FLAGS_timeout_ms >= 1 && FLAGS_timeout_ms <= 5000 && + FLAGS_deadline_ms >= 1 && FLAGS_deadline_ms <= 120000; +} + +struct Shared { + std::atomic failed{false}; + std::mutex error_mutex; + void Fail(const std::string& message) { + if (!failed.exchange(true)) { + std::lock_guard lock(error_mutex); + std::cerr << message << '\n'; + } + } +}; + +// Builds one synchronous RPC for the selected protocol. Both protocols carry +// the same request string and attachment and verify every echoed byte. +CallFn MakeCall(brpc::Channel* channel, const std::string& text, + const std::string& attachment) { + if (FLAGS_protocol == "baidu_std") { + auto stub = std::make_shared(channel); + return [stub, &text, &attachment](uint64_t request_id, int64_t timeout_ms, + std::string* error) { + benchmark_fb_pb::Request request; + request.set_request_id(request_id); + if (!FLAGS_omit_message) { + request.set_message(text); + } + benchmark_fb_pb::Response response; + brpc::Controller cntl; + cntl.set_timeout_ms(timeout_ms); + cntl.request_attachment().append(attachment); + stub->Echo(&cntl, &request, &response, nullptr); + if (cntl.Failed()) { + *error = "RPC failed: " + cntl.ErrorText(); + return kFailed; + } + if (!CheckPbResponse(response, request_id, text, attachment, cntl)) { + *error = "Response fields, optional string or attachment mismatch"; + return kFailed; + } + return kSuccess; + }; + } + // Generated stub methods route through Channel::FBCallMethod. + auto stub = std::make_shared(channel); + return [stub, &text, &attachment](uint64_t request_id, int64_t timeout_ms, + std::string* error) { + const Message request = MakeRequest(request_id, text); + Message response; + brpc::Controller cntl; + cntl.set_timeout_ms(timeout_ms); + cntl.request_attachment().append(attachment); + stub->Echo(&cntl, &request, &response, nullptr); + if (FLAGS_corrupt_request) { + // The generated generic RpcController::SetFailed(string) + // maps to -1. Timeouts and connection errors do not count. + if (!cntl.Failed() || cntl.ErrorCode() != -1 || + response.size() != 0 || !cntl.response_attachment().empty()) { + *error = "Expected server schema rejection; error=" + cntl.ErrorText(); + return kFailed; + } + return kRejected; + } + if (cntl.Failed()) { + *error = "RPC failed: " + cntl.ErrorText(); + return kFailed; + } + if (!CheckResponse(response, request_id, text, attachment, cntl)) { + *error = "Response schema, fields, optional string or attachment mismatch"; + return kFailed; + } + return kSuccess; + }; +} + +struct BthreadArg { + Shared* shared; + const std::function* body; + int index; +}; + +void* BthreadEntry(void* raw) { + std::unique_ptr arg(static_cast(raw)); + try { + (*arg->body)(arg->index); + } catch (const std::exception& error) { + arg->shared->Fail(std::string("Client worker failed: ") + error.what()); + } + return nullptr; +} + +void RunWorkers(Shared* shared, const std::function& body) { + if (FLAGS_use_bthread) { + std::vector ids(FLAGS_thread_num, INVALID_BTHREAD); + for (int i = 0; i < FLAGS_thread_num; ++i) { + auto* arg = new BthreadArg{shared, &body, i}; + if (bthread_start_background(&ids[i], nullptr, BthreadEntry, arg) != 0) { + delete arg; + ids[i] = INVALID_BTHREAD; + shared->Fail("Cannot start bthread worker"); + } + } + for (bthread_t id : ids) { + if (id != INVALID_BTHREAD) { + bthread_join(id, nullptr); + } + } + return; + } + std::vector workers; + workers.reserve(FLAGS_thread_num); + try { + for (int i = 0; i < FLAGS_thread_num; ++i) { + workers.emplace_back([shared, &body, i] { + try { + body(i); + } catch (const std::exception& error) { + shared->Fail(std::string("Client worker failed: ") + error.what()); + } + }); + } + } catch (const std::exception& error) { + shared->Fail(std::string("Cannot start client worker: ") + error.what()); + } + for (auto& thread : workers) { + thread.join(); + } +} + +// Bounded functional mode used by smoke.py. +int RunCount(const CallFn& call) { + const auto deadline = Clock::now() + std::chrono::milliseconds(FLAGS_deadline_ms); + std::atomic next{0}; + std::atomic successes{0}; + std::atomic expected_rejections{0}; + Shared shared; + RunWorkers(&shared, [&](int) { + while (!shared.failed.load()) { + const int index = next.fetch_add(1); + if (index >= FLAGS_request_count) { + break; + } + const int64_t remaining = std::chrono::duration_cast< + std::chrono::milliseconds>(deadline - Clock::now()).count(); + if (remaining <= 0) { + shared.Fail("Overall client deadline exceeded"); + break; + } + std::string error; + const CallResult result = call(static_cast(index) + 1, + std::min(FLAGS_timeout_ms, remaining), &error); + if (result == kFailed) { + shared.Fail(error); + break; + } + ++(result == kRejected ? expected_rejections : successes); + } + }); + const int completed = successes.load() + expected_rejections.load(); + if (completed != FLAGS_request_count) { + shared.Fail("Not all requested RPCs completed"); + } + std::cout << "{\"completed\":" << completed + << ",\"successes\":" << successes.load() + << ",\"expected_rejections\":" << expected_rejections.load() + << ",\"failures\":" << (shared.failed.load() ? 1 : 0) << "}\n"; + return shared.failed.load() ? 1 : 0; +} + +double CpuSeconds() { + rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec + usage.ru_stime.tv_sec + + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1e6; +} + +// Closed-loop benchmark: every caller keeps one RPC in flight for +// warmup_s + duration_s and records per-call latency in the measured window. +int RunTimed(const CallFn& call) { + Shared shared; + std::vector> latencies(FLAGS_thread_num); + std::atomic next_id{1}; + std::atomic measuring{false}; + std::atomic stop{false}; + std::atomic failures{0}; + std::thread timer([&] { + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_warmup_s)); + measuring = true; + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration_s)); + stop = true; + }); + // Wait for the warm-up boundary on this thread to sample CPU and time. + auto measure_begin = Clock::now(); + double cpu_begin = 0; + std::thread sampler([&] { + while (!measuring.load() && !stop.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + measure_begin = Clock::now(); + cpu_begin = CpuSeconds(); + }); + RunWorkers(&shared, [&](int index) { + std::vector& samples = latencies[index]; + samples.reserve(1 << 16); + while (!stop.load(std::memory_order_relaxed)) { + const bool measured = measuring.load(std::memory_order_relaxed); + const auto begin = Clock::now(); + std::string error; + const CallResult result = call(next_id.fetch_add(1, std::memory_order_relaxed), + FLAGS_timeout_ms, &error); + const auto end = Clock::now(); + if (result != kSuccess) { + if (failures.fetch_add(1) == 0) { + shared.Fail(error); + } + continue; + } + if (measured && !stop.load(std::memory_order_relaxed)) { + samples.push_back(static_cast(std::min( + std::chrono::duration_cast( + end - begin).count(), UINT32_MAX))); + } + } + }); + const auto measure_end = Clock::now(); + const double cpu_end = CpuSeconds(); + timer.join(); + sampler.join(); + std::vector all; + for (const auto& samples : latencies) { + all.insert(all.end(), samples.begin(), samples.end()); + } + std::sort(all.begin(), all.end()); + const double seconds = std::chrono::duration(measure_end - measure_begin).count(); + auto percentile = [&all](double p) -> uint32_t { + if (all.empty()) { + return 0; + } + const size_t rank = static_cast(p * (all.size() - 1) + 0.5); + return all[std::min(rank, all.size() - 1)]; + }; + double sum = 0; + for (uint32_t value : all) { + sum += value; + } + const double qps = seconds > 0 ? all.size() / seconds : 0; + const double payload = static_cast(FLAGS_omit_message ? 0 : FLAGS_request_size) + + FLAGS_attachment_size; + std::cout << std::fixed << std::setprecision(1) + << "{\"mode\":\"timed\",\"protocol\":\"" << FLAGS_protocol + << "\",\"connection_type\":\"" << FLAGS_connection_type + << "\",\"use_bthread\":" << (FLAGS_use_bthread ? "true" : "false") + << ",\"thread_num\":" << FLAGS_thread_num + << ",\"request_size\":" << FLAGS_request_size + << ",\"attachment_size\":" << FLAGS_attachment_size + << ",\"seconds\":" << seconds + << ",\"requests\":" << all.size() + << ",\"failures\":" << failures.load() + << ",\"qps\":" << qps + << ",\"payload_mib_per_s\":" << qps * payload / (1024.0 * 1024.0) + << ",\"avg_us\":" << (all.empty() ? 0 : sum / all.size()) + << ",\"p50_us\":" << percentile(0.50) + << ",\"p90_us\":" << percentile(0.90) + << ",\"p99_us\":" << percentile(0.99) + << ",\"p999_us\":" << percentile(0.999) + << ",\"max_us\":" << (all.empty() ? 0 : all.back()) + << ",\"client_cpu_cores\":" << std::setprecision(3) + << (seconds > 0 ? (cpu_end - cpu_begin) / seconds : 0) + << "}\n"; + return failures.load() == 0 && !all.empty() ? 0 : 1; +} + +int RunClient(const butil::EndPoint& endpoint) { + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = FLAGS_protocol; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = std::min(FLAGS_timeout_ms, 1000); + options.max_retry = 0; + if (channel.Init(endpoint, &options) != 0) { + std::cerr << "Cannot initialize " << FLAGS_protocol << " channel\n"; + return 1; + } + const std::string text = MakeBytes(FLAGS_omit_message ? 0 : FLAGS_request_size, 31); + const std::string attachment = MakeBytes(FLAGS_attachment_size, 17); + const CallFn call = MakeCall(&channel, text, attachment); + return FLAGS_duration_s > 0 ? RunTimed(call) : RunCount(call); +} +} // namespace + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + butil::EndPoint endpoint; + if (!ValidFlags(&endpoint)) { + std::cerr << "Invalid flags; --server=127.0.0.1:PORT is required unless " + "--allow_remote is set. See --help and README.md for bounded " + "parameter ranges.\n"; + return 2; + } + try { + return RunClient(endpoint); + } catch (const std::exception& error) { + std::cerr << "Client failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/example/benchmark_fb/echo.fbs b/example/benchmark_fb/echo.fbs new file mode 100644 index 0000000000..f7a40eae2b --- /dev/null +++ b/example/benchmark_fb/echo.fbs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace benchmark_fb; + +table Request { + request_id:ulong; + message:string; +} + +table Response { + request_id:ulong; + message:string; + attachment_size:ulong; +} + +rpc_service BenchmarkService { + Echo(Request):Response (id: 7); +} + +root_type Request; diff --git a/example/benchmark_fb/echo_pb.proto b/example/benchmark_fb/echo_pb.proto new file mode 100644 index 0000000000..bded8516a2 --- /dev/null +++ b/example/benchmark_fb/echo_pb.proto @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Protobuf control service with the same fields as echo.fbs, served over +// baidu_std so fb_rpc can be compared against an equivalent payload. +syntax = "proto2"; + +package benchmark_fb_pb; + +option cc_generic_services = true; + +message Request { + optional uint64 request_id = 1; + optional bytes message = 2; +} + +message Response { + optional uint64 request_id = 1; + optional bytes message = 2; + optional uint64 attachment_size = 3; +} + +service BenchmarkService { + rpc Echo(Request) returns (Response); +} diff --git a/example/benchmark_fb/server.cpp b/example/benchmark_fb/server.cpp new file mode 100644 index 0000000000..7989573272 --- /dev/null +++ b/example/benchmark_fb/server.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include "butil/config.h" +#include "butil/endpoint.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/errno.pb.h" +#include "brpc/server.h" +#include "echo.brpc.fb.h" +#include "echo_pb.pb.h" + +#if !BRPC_WITH_FLATBUFFERS +#error "benchmark_fb requires a FlatBuffers-enabled brpc library" +#endif + +DEFINE_string(listen_addr, "127.0.0.1:0", "IPv4 address; port 0 selects a free port. " + "A non-loopback address also needs --allow_remote"); +DEFINE_bool(allow_remote, false, "Allow a non-loopback --listen_addr for cross-host benchmarks"); +DEFINE_int32(duration_s, 60, "Exit after 1..86400 seconds, or earlier on SIGINT/SIGTERM"); +DEFINE_int32(max_concurrency, 32, "Maximum concurrent requests, 1..4096"); + +namespace { +const size_t kMaxBytes = 1024 * 1024; + +// Reserve the echoed string plus table overhead up front so a large reply +// does not grow the builder by repeated reallocate-and-copy. +size_t BuilderSize(const benchmark_fb::Request* input) { + const size_t text = input->message() ? input->message()->size() : 0; + return text + 256; +} + +class BenchmarkServiceImpl : public benchmark_fb::BenchmarkService { +public: + void Echo(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + if (!controller) { + return; + } + auto* cntl = static_cast(controller); + // The generated dispatcher also verifies the schema. Keep this guard + // for callers invoking the typed service method directly. + if (!request || !response || !request->Verify()) { + cntl->SetFailed(brpc::EREQUEST, "Invalid FlatBuffers Request"); + return; + } + const auto* input = request->GetRoot(); + if ((input->message() && input->message()->size() > kMaxBytes) || + cntl->request_attachment().size() > kMaxBytes) { + cntl->SetFailed(brpc::EREQUEST, "Example payload limit is 1 MiB per field"); + return; + } + brpc::flatbuffers::MessageBuilder builder(BuilderSize(input)); + // Preserve absent versus empty strings, including embedded zero bytes. + const auto text = input->message() ? + builder.CreateString(input->message()->data(), input->message()->size()) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(benchmark_fb::CreateResponse( + builder, input->request_id(), text, cntl->request_attachment().size())); + *response = builder.ReleaseMessage(); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +// Protobuf control service over baidu_std with the same echo semantics. +class PbBenchmarkServiceImpl : public benchmark_fb_pb::BenchmarkService { +public: + void Echo(google::protobuf::RpcController* controller, + const benchmark_fb_pb::Request* request, + benchmark_fb_pb::Response* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + auto* cntl = static_cast(controller); + if (request->message().size() > kMaxBytes || + cntl->request_attachment().size() > kMaxBytes) { + cntl->SetFailed(brpc::EREQUEST, "Example payload limit is 1 MiB per field"); + return; + } + response->set_request_id(request->request_id()); + if (request->has_message()) { + response->set_message(request->message()); + } + response->set_attachment_size(cntl->request_attachment().size()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; +} // namespace + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + butil::EndPoint endpoint; + if ((FLAGS_listen_addr.compare(0, 10, "127.0.0.1:") != 0 && !FLAGS_allow_remote) || + butil::str2endpoint(FLAGS_listen_addr.c_str(), &endpoint) != 0 || + FLAGS_duration_s < 1 || FLAGS_duration_s > 86400 || + FLAGS_max_concurrency < 1 || FLAGS_max_concurrency > 4096) { + std::cerr << "Use --listen_addr=127.0.0.1:PORT (or --allow_remote), " + "--duration_s=1..86400 and --max_concurrency=1..4096\n"; + return 2; + } + // brpc defaults SIGTERM to immediate termination. This example always + // enables graceful shutdown, including the smoke's owned-process cleanup. + if (GFLAGS_NAMESPACE::SetCommandLineOption( + "graceful_quit_on_sigterm", "true").empty()) { + std::cerr << "Cannot enable graceful SIGTERM handling\n"; + return 1; + } + BenchmarkServiceImpl service; + brpc::Server server; + if (server.AddFlatBuffersService(&service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + std::cerr << "Cannot register FlatBuffers service\n"; + return 1; + } + PbBenchmarkServiceImpl pb_service; + if (server.AddService(&pb_service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + std::cerr << "Cannot register protobuf control service\n"; + return 1; + } + brpc::ServerOptions options; + options.has_builtin_services = false; + options.enabled_protocols = "fb_rpc baidu_std"; + options.max_concurrency = FLAGS_max_concurrency; + if (server.Start(endpoint, &options) != 0) { + std::cerr << "Cannot start FlatBuffers server\n"; + return 1; + } + // Install brpc's termination handlers before publishing readiness. + brpc::IsAskedToQuit(); + std::cout << "BRPC_FB_READY " << server.listen_address() << std::endl; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(FLAGS_duration_s); + while (!brpc::IsAskedToQuit() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + const int stop_result = server.Stop(0); + const int join_result = server.Join(); + return stop_result == 0 && join_result == 0 ? 0 : 1; +} diff --git a/example/benchmark_fb/smoke.py b/example/benchmark_fb/smoke.py new file mode 100644 index 0000000000..5b9c621cfc --- /dev/null +++ b/example/benchmark_fb/smoke.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bounded functional smoke: no network dependency and no throughput measurement.""" + +import argparse +import json +import os +from pathlib import Path +import re +import selectors +import signal +import subprocess +import sys +import tempfile +import time + + +def remaining(deadline): + seconds = deadline - time.monotonic() + if seconds <= 0: + raise RuntimeError("smoke deadline exceeded") + return seconds + + +def wait_ready(process, deadline): + buffer = b"" + os.set_blocking(process.stdout.fileno(), False) + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while True: + if process.poll() is not None: + raise RuntimeError("server exited before readiness") + events = selector.select(min(0.2, remaining(deadline))) + if not events: + continue + chunk = os.read(process.stdout.fileno(), 4096) + if not chunk: + raise RuntimeError("server closed stdout before readiness") + buffer += chunk + if len(buffer) > 65536: + raise RuntimeError("unexpectedly large server readiness output") + while b"\n" in buffer: + line, buffer = buffer.split(b"\n", 1) + match = re.fullmatch(rb"BRPC_FB_READY (127\.0\.0\.1):([0-9]{1,5})", line) + if match: + port = int(match.group(2)) + if not 1 <= port <= 65535: + raise RuntimeError("server published an invalid port") + return "127.0.0.1:" + str(port) + + +def run_client(binary, endpoint, deadline, *, count, size, attachment, + threads=1, omit=False, corrupt=False, connection="single"): + command = [ + str(binary), "--server=" + endpoint, + "--request_count=" + str(count), "--thread_num=" + str(threads), + "--request_size=" + str(size), "--attachment_size=" + str(attachment), + "--connection_type=" + connection, + "--omit_message=" + str(omit).lower(), + "--corrupt_request=" + str(corrupt).lower(), + "--timeout_ms=1000", "--deadline_ms=5000", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=min(8.0, remaining(deadline)), check=False) + if result.returncode != 0: + raise RuntimeError("client failed ({}):\n{}\n{}".format( + result.returncode, result.stdout, result.stderr)) + try: + report = json.loads(result.stdout) + except (ValueError, TypeError) as error: + raise RuntimeError("invalid client summary: " + result.stdout) from error + expected = {"completed": count, "successes": 0 if corrupt else count, + "expected_rejections": count if corrupt else 0, "failures": 0} + if report != expected: + raise RuntimeError("unexpected client result: " + repr(report)) + + +def stop_owned_server(process): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + raise RuntimeError("server required SIGKILL instead of a clean Stop/Join") + if process.returncode != 0: + raise RuntimeError("server exited with status " + str(process.returncode)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, help="benchmark_fb_server executable") + parser.add_argument("--client", required=True, help="benchmark_fb_client executable") + args = parser.parse_args() + process = None + failure = None + deadline = time.monotonic() + 35 + with tempfile.TemporaryFile(mode="w+b") as server_log: + try: + server = Path(args.server).resolve(strict=True) + client = Path(args.client).resolve(strict=True) + process = subprocess.Popen( + [str(server), "--listen_addr=127.0.0.1:0", "--duration_s=45", + "--max_concurrency=8"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=server_log, + start_new_session=True, + ) + endpoint = wait_ready(process, min(deadline, time.monotonic() + 10)) + run_client(client, endpoint, deadline, + count=6, size=8193, attachment=257, threads=2) + run_client(client, endpoint, deadline, + count=2, size=0, attachment=31, connection="pooled") + run_client(client, endpoint, deadline, + count=2, size=0, attachment=0, omit=True, connection="short") + run_client(client, endpoint, deadline, + count=2, size=17, attachment=5, corrupt=True) + # Keep the same server alive after the malformed requests. + run_client(client, endpoint, deadline, count=3, size=33, attachment=9) + if process.poll() is not None: + raise RuntimeError("server exited during the smoke") + except (Exception, KeyboardInterrupt) as error: + failure = str(error) or type(error).__name__ + finally: + if process is not None: + try: + stop_owned_server(process) + except Exception as error: + failure = failure or str(error) + if process.stdout is not None: + process.stdout.close() + if failure: + server_log.seek(0) + log = server_log.read(65536).decode("utf-8", errors="replace") + sys.stderr.write("benchmark_fb smoke failed: " + failure + "\n" + log) + if failure: + return 1 + print("benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown") + return 0 + + +def handle_termination(signum, frame): + raise KeyboardInterrupt() + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, handle_termination) + sys.exit(main()) diff --git a/src/brpc/CMakeLists.txt b/src/brpc/CMakeLists.txt index 03e45da465..9378d0c8e2 100644 --- a/src/brpc/CMakeLists.txt +++ b/src/brpc/CMakeLists.txt @@ -251,6 +251,8 @@ file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*. file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/thrift*.cpp") if(NOT WITH_FLATBUFFERS) list(FILTER BRPC_SOURCES EXCLUDE REGEX "/brpc/flatbuffers/.*\\.cpp$") + list(REMOVE_ITEM BRPC_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/policy/flatbuffers_protocol.cpp") endif() file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/event_dispatcher_*.cpp") diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index de0a1bde49..ee5aa64175 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -40,6 +40,9 @@ #include "brpc/transport_factory.h" #include "brpc/details/controller_private_accessor.h" #include "brpc/details/ssl_helper.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/flatbuffers/message.h" +#endif namespace brpc { @@ -484,6 +487,29 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, const google::protobuf::Message* request, google::protobuf::Message* response, google::protobuf::Closure* done) { +#if BRPC_WITH_FLATBUFFERS + static_cast(controller_base)->_flatbuffers_method = nullptr; +#endif + CallMethodInternal(method, controller_base, request, response, done, false); +} + +#if BRPC_WITH_FLATBUFFERS +void Channel::FBCallMethod(const flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const flatbuffers::Message* request, + flatbuffers::Message* response, + google::protobuf::Closure* done) { + static_cast(controller_base)->_flatbuffers_method = method; + CallMethodInternal(nullptr, controller_base, request, response, done, true); +} +#endif + +void Channel::CallMethodInternal( + const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done, bool is_fb) { const int64_t start_send_real_us = butil::gettimeofday_us(); Controller* cntl = static_cast(controller_base); cntl->OnRPCBegin(start_send_real_us); @@ -538,12 +564,17 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, return; } cntl->set_used_by_rpc(); + const bool is_fb_protocol = (_options.protocol == PROTOCOL_FLATBUFFERS_RPC); if (cntl->_sender == nullptr && IsTraceable(Span::tls_parent().get())) { const int64_t start_send_us = butil::cpuwide_time_us(); std::string method_name; - if (_get_method_name) { + if (_get_method_name && is_fb == is_fb_protocol) { method_name = butil::EnsureString(_get_method_name(method, cntl)); +#if BRPC_WITH_FLATBUFFERS + } else if (is_fb && cntl->_flatbuffers_method) { + method_name = cntl->_flatbuffers_method->full_name(); +#endif } else if (method) { method_name = butil::EnsureString(method->full_name()); } else { @@ -591,6 +622,33 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, // Share the lb with controller. cntl->_lb = _lb; + const char* entry_error = nullptr; + if (is_fb != is_fb_protocol) { + entry_error = is_fb ? "FlatBuffers calls require the fb_rpc protocol" : + "The fb_rpc protocol requires FBCallMethod"; + } +#if BRPC_WITH_FLATBUFFERS + else if (is_fb) { + if (!cntl->_flatbuffers_method || !request || !response) { + entry_error = "FlatBuffers method, request and response " + "must not be null"; + } else if (_options.auth != nullptr) { + // An authenticated shared socket may skip the packer's auth check. + entry_error = "The fb_rpc protocol does not support authentication"; + } else if (!cntl->_request_streams.empty() || + !cntl->_response_streams.empty()) { + entry_error = "The fb_rpc protocol does not support streams"; + } + } +#endif + if (entry_error) { + // A custom retry policy must not pack an unserialized request. Keep + // the normal completion path; callbacks may delete the controller. + cntl->set_max_retry(0); + cntl->SetFailed(EINVAL, "%s", entry_error); + return cntl->HandleSendFailed(); + } + // Ensure that serialize_request is done before pack_request in all // possible executions, including: // HandleSendFailed => OnVersionedRPCReturned => IssueRPC(pack_request) diff --git a/src/brpc/channel.h b/src/brpc/channel.h index a778a1bd2a..ce490e203f 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -23,6 +23,7 @@ // on internal structures, use opaque pointers instead. #include // std::ostream +#include "butil/config.h" #include "bthread/errno.h" // Redefine errno #include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr #include "butil/ptr_container.h" @@ -38,6 +39,9 @@ #include "brpc/naming_service_filter.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/flatbuffers/service.h" +#endif namespace brpc { @@ -176,7 +180,11 @@ struct ChannelOptions { // channel.Init("bns://rdev.matrix.all", "rr", nullptr/*default options*/); // MyService_Stub stub(&channel); // stub.MyMethod(&controller, &request, &response, nullptr); -class Channel : public ChannelBase { +class Channel : public ChannelBase +#if BRPC_WITH_FLATBUFFERS + , public flatbuffers::RpcChannel +#endif +{ friend class Controller; friend class SelectiveChannel; public: @@ -228,6 +236,15 @@ friend class SelectiveChannel; google::protobuf::Message* response, google::protobuf::Closure* done); +#if BRPC_WITH_FLATBUFFERS + // The descriptor is borrowed and must remain valid until the RPC finishes. + void FBCallMethod(const flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const flatbuffers::Message* request, + flatbuffers::Message* response, + google::protobuf::Closure* done) override; +#endif + // Get current options. const ChannelOptions& options() const { return _options; } @@ -238,6 +255,13 @@ friend class SelectiveChannel; int CheckHealth(); +private: + void CallMethodInternal(const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done, bool is_fb); + protected: bool SingleServer() const { return _lb.get() == nullptr; } diff --git a/src/brpc/controller.cpp b/src/brpc/controller.cpp index 4fff9fd2f4..ce5c8e933d 100644 --- a/src/brpc/controller.cpp +++ b/src/brpc/controller.cpp @@ -510,6 +510,9 @@ void Controller::ResetPods() { _accessed = nullptr; _pack_request = nullptr; _method = nullptr; +#if BRPC_WITH_FLATBUFFERS + _flatbuffers_method = nullptr; +#endif _auth = nullptr; _idl_names = idl_single_req_single_res; _idl_result = IDL_VOID_RESULT; diff --git a/src/brpc/controller.h b/src/brpc/controller.h index c05dbb75a7..25ebcb0999 100644 --- a/src/brpc/controller.h +++ b/src/brpc/controller.h @@ -26,6 +26,7 @@ #include // Users often need gflags #include #include +#include "butil/config.h" #include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr #include "bthread/errno.h" // Redefine errno #include "butil/endpoint.h" // butil::EndPoint @@ -64,6 +65,11 @@ struct x509_st; } namespace brpc { +#if BRPC_WITH_FLATBUFFERS +namespace flatbuffers { +class MethodDescriptor; +} +#endif class Span; class Server; class SharedLoadBalancer; @@ -336,6 +342,12 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); ConnectionType connection_type() const { return _connection_type; } // Get the called method. May-be nullptr for non-pb services. const google::protobuf::MethodDescriptor* method() const { return _method; } +#if BRPC_WITH_FLATBUFFERS + // Borrowed descriptor. The caller must keep it alive until the RPC ends. + const flatbuffers::MethodDescriptor* flatbuffers_method() const { + return _flatbuffers_method; + } +#endif // Get the controllers for accessing sub channels in combo channels. // Ordinary channel: @@ -961,6 +973,9 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Fields will be used when making requests Protocol::PackRequest _pack_request; const google::protobuf::MethodDescriptor* _method; +#if BRPC_WITH_FLATBUFFERS + const flatbuffers::MethodDescriptor* _flatbuffers_method; +#endif const Authenticator* _auth; butil::IOBuf _request_buf; IdlNames _idl_names; diff --git a/src/brpc/details/controller_private_accessor.h b/src/brpc/details/controller_private_accessor.h index d752d7fe5e..12200e3bc7 100644 --- a/src/brpc/details/controller_private_accessor.h +++ b/src/brpc/details/controller_private_accessor.h @@ -149,6 +149,11 @@ class ControllerPrivateAccessor { void set_method(const google::protobuf::MethodDescriptor* method) { _cntl->_method = method; } +#if BRPC_WITH_FLATBUFFERS + void set_flatbuffers_method(const flatbuffers::MethodDescriptor* method) + { _cntl->_flatbuffers_method = method; } +#endif + void set_readable_progressive_attachment(ReadableProgressiveAttachment* s) { _cntl->_rpa.reset(s); } diff --git a/src/brpc/details/server_private_accessor.h b/src/brpc/details/server_private_accessor.h index 0e6e4fbba8..90448a2db5 100644 --- a/src/brpc/details/server_private_accessor.h +++ b/src/brpc/details/server_private_accessor.h @@ -76,6 +76,15 @@ class ServerPrivateAccessor { return _server->FindMethodPropertyByNameAndIndex(service_name, method_index); } +#if BRPC_WITH_FLATBUFFERS + const Server::FlatBuffersMethodProperty* + FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, + int32_t method_index) const { + return _server->FindFlatBuffersMethodPropertyByIndex( + service_index, method_index); + } +#endif + const Server::ServiceProperty* FindServicePropertyByFullName(const butil::StringPiece& fullname) const { return _server->FindServicePropertyByFullName(fullname); diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 4cce93ed0c..0ab7a33405 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -71,6 +71,9 @@ #include "brpc/protocol.h" #include "brpc/policy/rdma_handshake_protocol.h" #include "brpc/policy/baidu_rpc_protocol.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/policy/flatbuffers_protocol.h" +#endif #include "brpc/policy/http_rpc_protocol.h" #include "brpc/policy/http2_rpc_protocol.h" #include "brpc/policy/hulu_pbrpc_protocol.h" @@ -456,6 +459,17 @@ static void GlobalInitializeOrDieImpl() { exit(1); } +#if BRPC_WITH_FLATBUFFERS + Protocol flatbuffers_protocol = { + ParseFlatBuffersMessage, SerializeFlatBuffersRequest, PackFlatBuffersRequest, + ProcessFlatBuffersRequest, ProcessFlatBuffersResponse, + VerifyFlatBuffersRequest, nullptr, GetFlatBuffersMethodName, + CONNECTION_TYPE_ALL, "fb_rpc" }; + if (RegisterProtocol(PROTOCOL_FLATBUFFERS_RPC, flatbuffers_protocol) != 0) { + exit(1); + } +#endif + Protocol streaming_protocol = { ParseStreamingMessage, nullptr, nullptr, ProcessStreamingMessage, ProcessStreamingMessage, diff --git a/src/brpc/options.proto b/src/brpc/options.proto index 13b8b682e6..23f28c0a9e 100644 --- a/src/brpc/options.proto +++ b/src/brpc/options.proto @@ -67,6 +67,7 @@ enum ProtocolType { PROTOCOL_H2 = 27; PROTOCOL_COUCHBASE = 28; PROTOCOL_MYSQL = 29; // Client side only + PROTOCOL_FLATBUFFERS_RPC = 30; } enum CompressType { diff --git a/src/brpc/policy/flatbuffers_protocol.cpp b/src/brpc/policy/flatbuffers_protocol.cpp new file mode 100644 index 0000000000..91f7e7cd46 --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.cpp @@ -0,0 +1,476 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/policy/flatbuffers_protocol.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include "butil/logging.h" +#include "butil/macros.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/socket.h" +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" +#include "brpc/details/usercode_backup_pool.h" + +extern "C" void bthread_assign_data(void* data); + +namespace brpc { +namespace policy { +namespace { + +const size_t kHeaderSize = 12; +const size_t kRequestMetaSize = 24; +const size_t kResponseMetaSize = 20; + +// The header is big-endian; the fixed metadata prefix is little-endian, as in +// the experimental FRPC format on x86/ARM. meta_size permits appended fields. +// Bytewise access also handles fragmented or unaligned input without packed +// structs, aliasing violations or host-endian dependencies. +uint32_t Load32(const unsigned char* p, bool big_endian = false) { + uint32_t result = 0; + for (size_t i = 0; i < 4; ++i) { + result |= static_cast(p[i]) << (8 * (big_endian ? 3 - i : i)); + } + return result; +} + +uint64_t Load64(const unsigned char* p) { + return Load32(p) | (static_cast(Load32(p + 4)) << 32); +} + +void Store32(unsigned char* p, uint32_t value, bool big_endian = false) { + for (size_t i = 0; i < 4; ++i) { + p[i] = static_cast(value >> (8 * (big_endian ? 3 - i : i))); + } +} + +void Store64(unsigned char* p, uint64_t value) { + Store32(p, static_cast(value)); + Store32(p + 4, static_cast(value >> 32)); +} + +bool ValidSizes(size_t message_size, size_t attachment_size, size_t meta_size) { + const size_t limit = static_cast(std::numeric_limits::max()); + return message_size <= limit && attachment_size <= limit && + meta_size <= limit && message_size <= limit - meta_size && + attachment_size <= limit - meta_size - message_size && + meta_size + message_size + attachment_size <= + static_cast(FLAGS_max_body_size); +} + +void PackHeader(unsigned char* header, size_t meta_size, size_t payload_size) { + memcpy(header, "FRPC", 4); + Store32(header + 4, static_cast(meta_size + payload_size), true); + Store32(header + 8, static_cast(meta_size), true); +} + +bool SerializePayload(const flatbuffers::Message& message, butil::IOBuf* out) { + butil::IOBuf body; + if (!message.append_msg_to_iobuf(body)) { + return false; + } + body.pop_front(message.get_meta_size()); + if (body.size() != message.size()) { + return false; + } + out->append(body); + return true; +} + +bool ValidPayloadSizes(uint32_t message_size, uint32_t attachment_size, + size_t payload_size) { + const uint32_t limit = static_cast(std::numeric_limits::max()); + return message_size <= limit && attachment_size <= limit && + attachment_size <= payload_size && + message_size == payload_size - attachment_size; +} + +// One closure owns the request, response, controller and concurrency accounting +// until the application completes, including asynchronous service methods. +class FlatBuffersCall : public google::protobuf::Closure { +public: + FlatBuffersCall(uint64_t id, int64_t received_us) + : correlation_id(id), received_us(received_us), cntl(new Controller), + service(nullptr), method(nullptr), status(nullptr) {} + + ~FlatBuffersCall() override { + { + ConcurrencyRemover remover(status, cntl.get(), received_us); + } + cntl->CallAfterRpcResp(&request, &response); + } + + void Run() override { + std::unique_ptr self(this); + ControllerPrivateAccessor accessor(cntl.get()); + Socket* socket = accessor.get_sending_socket(); + if (cntl->IsCloseConnection()) { + socket->SetFailed(); + return; + } + + butil::IOBuf payload; + size_t attachment_size = 0; + if (!cntl->Failed()) { + if (cntl->response_compress_type() != COMPRESS_TYPE_NONE || + cntl->response_checksum_type() != CHECKSUM_TYPE_NONE) { + cntl->SetFailed(ERESPONSE, "FRPC does not support compression or checksums"); + } else if (!SerializePayload(response, &payload)) { + cntl->SetFailed(ERESPONSE, "Empty or invalid FlatBuffers response"); + } else { + attachment_size = cntl->response_attachment().size(); + if (!ValidSizes(payload.size(), attachment_size, kResponseMetaSize)) { + cntl->SetFailed(ERESPONSE, "FlatBuffers response is too large"); + } + } + } + if (cntl->Failed()) { + payload.clear(); + attachment_size = 0; + } + + unsigned char header[kHeaderSize + kResponseMetaSize] = {}; + PackHeader(header, kResponseMetaSize, payload.size() + attachment_size); + Store32(header + 12, static_cast(cntl->ErrorCode())); + Store32(header + 16, static_cast(payload.size())); + Store32(header + 20, static_cast(attachment_size)); + Store64(header + 24, correlation_id); + butil::IOBuf wire; + if (wire.append(header, sizeof(header)) != 0) { + socket->SetFailed(ENOMEM, "Fail to allocate FlatBuffers response header"); + return; + } + wire.append(payload); + if (attachment_size) { + wire.append(cntl->response_attachment()); + } + Socket::WriteOptions options; + options.ignore_eovercrowded = true; + if (socket->Write(&wire, &options) != 0) { + cntl->SetFailed(errno, "Fail to write FlatBuffers response"); + } + } + + static void Invoke(void* arg) { + FlatBuffersCall* call = static_cast(arg); + call->service->FBCallMethod(call->method, call->cntl.get(), + &call->request, &call->response, call); + } + + uint64_t correlation_id; + int64_t received_us; + std::unique_ptr cntl; + flatbuffers::Message request; + flatbuffers::Message response; + flatbuffers::Service* service; + const flatbuffers::MethodDescriptor* method; + MethodStatus* status; + +private: + DISALLOW_COPY_AND_ASSIGN(FlatBuffersCall); +}; + +} // namespace + +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket*, bool, + const void*) { + unsigned char header[kHeaderSize] = {}; + const size_t n = source->copy_to(header, sizeof(header)); + if (memcmp(header, "FRPC", std::min(n, size_t(4))) != 0) { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + if (n < sizeof(header)) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + const uint32_t body_size = Load32(header + 4, true); + const uint32_t meta_size = Load32(header + 8, true); + if (body_size > static_cast(FLAGS_max_body_size)) { + return MakeParseError(PARSE_ERROR_TOO_BIG_DATA); + } + if (meta_size > body_size) { + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG, + "FRPC metadata exceeds frame body"); + } + if (source->size() - kHeaderSize < body_size) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + MostCommonMessage* message = MostCommonMessage::Get(); + if (!message) { + return MakeParseError(PARSE_ERROR_NO_RESOURCE); + } + source->pop_front(kHeaderSize); + source->cutn(&message->meta, meta_size); + source->cutn(&message->payload, body_size - meta_size); + return MakeMessage(message); +} + +bool VerifyFlatBuffersRequest(const InputMessageBase* message) { + const Server* server = static_cast(message->arg()); + // No credential format has been defined for FRPC. Never silently bypass + // authentication configured for the server's other protocols. + return server->options().auth == nullptr; +} + +void ProcessFlatBuffersRequest(InputMessageBase* message_base) { + DestroyingPtr message( + static_cast(message_base)); + SocketUniquePtr socket_guard(message->ReleaseSocket()); + Socket* socket = socket_guard.get(); + const Server* server = static_cast(message->arg()); + ScopedNonServiceError non_service_error(server); + unsigned char meta[kRequestMetaSize]; + if (message->meta.copy_to(meta, sizeof(meta)) != sizeof(meta)) { + socket->SetFailed(EREQUEST, "Truncated FlatBuffers request metadata"); + return; + } + const uint32_t service_id = Load32(meta); + const uint32_t method_id = Load32(meta + 4); + const uint32_t message_size = Load32(meta + 8); + const uint32_t attachment_size = Load32(meta + 12); + auto call = std::make_unique(Load64(meta + 16), + message->received_us()); + Controller* cntl = call->cntl.get(); + ControllerPrivateAccessor accessor(cntl); + ServerPrivateAccessor server_accessor(server); + const bool security_mode = server->options().security_mode() && + socket->user() == server_accessor.acceptor(); + cntl->set_rpc_received_us(message->received_us()); + accessor.set_server(server) + .set_security_mode(security_mode) + .set_peer_id(socket->id()) + .set_remote_side(socket->remote_side()) + .set_local_side(socket->local_side()) + .set_auth_context(socket->auth_context()) + .set_request_protocol(PROTOCOL_FLATBUFFERS_RPC) + .set_begin_time_us(message->received_us()) + .move_in_server_receiving_sock(socket_guard); + if (server->thread_local_options().thread_local_data_factory) { + bthread_assign_data((void*)&server->thread_local_options()); + } + + do { + if (server->options().auth) { + cntl->SetFailed(ERPCAUTH, "FRPC does not support authentication"); + break; + } + if (RejectNonBuiltinAccessFromInternalPort(cntl, *server)) { + break; + } + if (!server->IsRunning()) { + cntl->SetFailed(ELOGOFF, "Server is stopping"); + break; + } + if (!server_accessor.AddConcurrency(cntl)) { + cntl->SetFailed(ELIMIT, "Reached server's max_concurrency"); + break; + } + if (FLAGS_usercode_in_pthread && TooManyUserCode()) { + cntl->SetFailed(ELIMIT, "Too many user code tasks"); + break; + } + if (!message_size || !ValidPayloadSizes(message_size, attachment_size, + message->payload.size())) { + cntl->SetFailed(EREQUEST, "Invalid FlatBuffers request sizes"); + break; + } + if (method_id > static_cast(std::numeric_limits::max())) { + cntl->SetFailed(ENOMETHOD, "Invalid FlatBuffers method ID"); + break; + } + const Server::FlatBuffersMethodProperty* property = + server_accessor.FindFlatBuffersMethodPropertyByIndex(service_id, method_id); + if (!property) { + cntl->SetFailed(ENOMETHOD, "Unknown FlatBuffers service=%u method=%u", + service_id, method_id); + break; + } + call->service = property->service; + call->method = property->method; + accessor.set_flatbuffers_method(property->method); + if (socket->is_overcrowded() && !server->options().ignore_eovercrowded && + !property->ignore_eovercrowded) { + cntl->SetFailed(EOVERCROWDED, "FlatBuffers connection is overcrowded"); + break; + } + non_service_error.release(); + if (property->status) { + call->status = property->status; + if (!call->status->OnRequested(nullptr, cntl)) { + cntl->SetFailed(ELIMIT, "Reached method's max_concurrency"); + break; + } + } + if (!server->AcceptRequest(cntl)) { + break; + } + butil::IOBuf payload; + message->payload.cutn(&payload, message_size); + if (!call->request.parse_msg_from_iobuf(payload, message_size, 0)) { + cntl->SetFailed(EREQUEST, "Fail to parse FlatBuffers request"); + break; + } + cntl->request_attachment().swap(message->payload); + message.reset(); + if (FLAGS_usercode_in_pthread) { + RunUserCode(&FlatBuffersCall::Invoke, call.release()); + } else { + FlatBuffersCall::Invoke(call.release()); + } + return; + } while (false); + call.release()->Run(); +} + +void ProcessFlatBuffersResponse(InputMessageBase* message_base) { + DestroyingPtr message( + static_cast(message_base)); + unsigned char meta[kResponseMetaSize]; + if (message->meta.copy_to(meta, sizeof(meta)) != sizeof(meta)) { + if (message->socket()) { + message->socket()->SetFailed(ERESPONSE, "Truncated FlatBuffers response metadata"); + } + return; + } + const uint32_t error_code = Load32(meta); + const uint32_t message_size = Load32(meta + 4); + const uint32_t attachment_size = Load32(meta + 8); + const bthread_id_t correlation_id = {Load64(meta + 12)}; + Controller* cntl = nullptr; + const int rc = bthread_id_lock(correlation_id, reinterpret_cast(&cntl)); + if (rc != 0) { + return; + } + // A live correlation id alone does not prove that this response belongs + // to the attempt: it must also arrive on the socket the attempt used. + const SocketId response_socket_id = + message->socket() ? message->socket()->id() : INVALID_SOCKET_ID; + if (cntl == nullptr || !ControllerPrivateAccessor(cntl) + .does_response_match_request_socket(correlation_id, response_socket_id)) { + LOG(WARNING) << "correlation_id=" << correlation_id.value + << " of the FlatBuffers response from socket " + << response_socket_id + << " does not match a request sent over it, drop it"; + CHECK_EQ(0, bthread_id_unlock(correlation_id)); + return; + } + ControllerPrivateAccessor accessor(cntl); + const int saved_error = cntl->ErrorCode(); + cntl->set_rpc_received_us(message->received_us()); + do { + if (!ValidPayloadSizes(message_size, attachment_size, message->payload.size())) { + cntl->SetFailed(ERESPONSE, "Invalid FlatBuffers response sizes"); + break; + } + if (error_code) { + if (message_size || attachment_size) { + cntl->SetFailed(ERESPONSE, "FlatBuffers error response has a payload"); + break; + } + cntl->SetFailed(static_cast(error_code), "FlatBuffers server error"); + break; + } + if (!message_size) { + cntl->SetFailed(ERESPONSE, "Empty FlatBuffers response"); + break; + } + butil::IOBuf payload; + message->payload.cutn(&payload, message_size); + if (cntl->response()) { + if (cntl->response()->GetDescriptor() != flatbuffers::Message::descriptor()) { + cntl->SetFailed(ERESPONSE, "Response is not a FlatBuffers Message"); + break; + } + flatbuffers::Message response; + if (!response.parse_msg_from_iobuf(payload, message_size, 0)) { + cntl->SetFailed(ERESPONSE, "Fail to parse FlatBuffers response"); + break; + } + *static_cast(cntl->response()) = std::move(response); + } + cntl->response_attachment().swap(message->payload); + } while (false); + message.reset(); + accessor.OnResponse(correlation_id, saved_error); +} + +void SerializeFlatBuffersRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request) { + if (!request || request->GetDescriptor() != flatbuffers::Message::descriptor()) { + cntl->SetFailed(EREQUEST, "Request is not a FlatBuffers Message"); + return; + } + if (cntl->request_compress_type() != COMPRESS_TYPE_NONE || + cntl->request_checksum_type() != CHECKSUM_TYPE_NONE) { + cntl->SetFailed(EREQUEST, "FRPC does not support compression or checksums"); + return; + } + if (!SerializePayload(*static_cast(request), buf)) { + cntl->SetFailed(EREQUEST, "Empty or invalid FlatBuffers request"); + } +} + +void PackFlatBuffersRequest(butil::IOBuf* buf, SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* cntl, const butil::IOBuf& request, + const Authenticator* auth) { + const flatbuffers::MethodDescriptor* fb_method = cntl->flatbuffers_method(); + if (method || !fb_method || fb_method->index() < 0) { + cntl->SetFailed(ENOMETHOD, "Use Channel::FBCallMethod with a FlatBuffers method"); + return; + } + if (auth) { + cntl->SetFailed(EREQUEST, "FRPC does not support authentication"); + return; + } + const size_t attachment_size = cntl->request_attachment().size(); + if (!request.size() || !ValidSizes(request.size(), attachment_size, kRequestMetaSize)) { + cntl->SetFailed(EREQUEST, "Invalid FlatBuffers request size"); + return; + } + unsigned char header[kHeaderSize + kRequestMetaSize] = {}; + PackHeader(header, kRequestMetaSize, request.size() + attachment_size); + Store32(header + 12, fb_method->service()->index()); + Store32(header + 16, static_cast(fb_method->index())); + Store32(header + 20, static_cast(request.size())); + Store32(header + 24, static_cast(attachment_size)); + Store64(header + 28, correlation_id); + if (buf->append(header, sizeof(header)) != 0) { + cntl->SetFailed(ENOMEM, "Fail to allocate FlatBuffers request header"); + return; + } + buf->append(request); + buf->append(cntl->request_attachment()); +} + +const std::string& GetFlatBuffersMethodName( + const google::protobuf::MethodDescriptor*, const Controller* cntl) { + static const std::string empty; + return cntl->flatbuffers_method() ? cntl->flatbuffers_method()->full_name() : empty; +} + +} // namespace policy +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/policy/flatbuffers_protocol.h b/src/brpc/policy/flatbuffers_protocol.h new file mode 100644 index 0000000000..e3e0065b6d --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_POLICY_FLATBUFFERS_PROTOCOL_H +#define BRPC_POLICY_FLATBUFFERS_PROTOCOL_H + +#include "butil/config.h" + +#if BRPC_WITH_FLATBUFFERS +#include "brpc/protocol.h" + +namespace brpc { +namespace policy { + +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket* socket, + bool read_eof, const void* arg); +void ProcessFlatBuffersRequest(InputMessageBase* msg); +void ProcessFlatBuffersResponse(InputMessageBase* msg); +bool VerifyFlatBuffersRequest(const InputMessageBase* msg); +void SerializeFlatBuffersRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request); +void PackFlatBuffersRequest(butil::IOBuf* buf, SocketMessage** user_message, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* cntl, const butil::IOBuf& request, + const Authenticator* auth); +const std::string& GetFlatBuffersMethodName( + const google::protobuf::MethodDescriptor* method, const Controller* cntl); + +} // namespace policy +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_POLICY_FLATBUFFERS_PROTOCOL_H diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 19d58ac422..ec6d02d167 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -83,6 +83,10 @@ #include "brpc/rdma/rdma_helper.h" #include "brpc/baidu_master_service.h" #include "brpc/transport_factory.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include "brpc/flatbuffers/service.h" +#endif inline std::ostream& operator<<(std::ostream& os, const timeval& tm) { const char old_fill = os.fill(); @@ -114,6 +118,58 @@ const char* status_str(Server::Status s) { return "UNKNOWN_STATUS"; } +#if BRPC_WITH_FLATBUFFERS +struct Server::FlatBuffersServiceMap { + struct Method { + Method(flatbuffers::Service* service, + const flatbuffers::MethodDescriptor* descriptor) + : status(std::make_unique()) + , property{service, descriptor, status.get(), false} {} + + std::unique_ptr status; + FlatBuffersMethodProperty property; + AdaptiveMaxConcurrency max_concurrency; + + private: + DISALLOW_COPY_AND_ASSIGN(Method); + }; + struct Service { + Service(flatbuffers::Service* svc, + const flatbuffers::ServiceDescriptor* desc) + : service(svc) + , descriptor(desc) + , ownership(SERVER_DOESNT_OWN_SERVICE) {} + ~Service() { + methods.clear(); + if (ownership == SERVER_OWNS_SERVICE) { + delete service; + } + } + + flatbuffers::Service* service; + const flatbuffers::ServiceDescriptor* descriptor; + ServiceOwnership ownership; + std::map > methods; + + private: + DISALLOW_COPY_AND_ASSIGN(Service); + }; + typedef std::map > ServiceMap; + ServiceMap services; + + Method* FindMethodByFullName(const butil::StringPiece& fullname) const { + for (const auto& service : services) { + for (const auto& method : service.second->methods) { + if (method.second->property.method->full_name() == fullname) { + return method.second.get(); + } + } + } + return nullptr; + } +}; +#endif + butil::static_atomic g_running_server_count = BUTIL_STATIC_ATOMIC_INIT(0); // Following services may have security issues and are disabled by default. @@ -361,6 +417,19 @@ void* Server::UpdateDerivedVars(void* arg) { it->second.status->Expose(mprefix); } } +#if BRPC_WITH_FLATBUFFERS + if (server->_flatbuffers_services) { + for (const auto& service : server->_flatbuffers_services->services) { + for (const auto& method : service.second->methods) { + mprefix.resize(prefix.size()); + mprefix.push_back('_'); + bvar::to_underscored_name( + &mprefix, method.second->property.method->full_name()); + method.second->status->Expose(mprefix); + } + } + } +#endif if (server->options().baidu_master_service) { server->options().baidu_master_service->Expose(prefix); } @@ -1105,6 +1174,27 @@ int Server::StartInternal(const butil::EndPoint& endpoint, it->second.max_concurrency.SetConcurrencyLimiter(cl); } } +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (const auto& service : _flatbuffers_services->services) { + for (const auto& entry : service.second->methods) { + auto& method = *entry.second; + const AdaptiveMaxConcurrency* amc = &method.max_concurrency; + if (amc->type() == AdaptiveMaxConcurrency::UNLIMITED()) { + amc = &_options.method_max_concurrency; + } + ConcurrencyLimiter* cl = nullptr; + if (!CreateConcurrencyLimiter(*amc, &cl)) { + LOG(ERROR) << "Fail to create ConcurrencyLimiter for " + << method.property.method->full_name(); + return -1; + } + method.status->SetConcurrencyLimiter(cl); + method.max_concurrency.SetConcurrencyLimiter(cl); + } + } + } +#endif if (0 != SetServiceMaxConcurrency(_options.nshead_service)) { return -1; } @@ -1426,6 +1516,22 @@ int Server::AddServiceInternal(google::protobuf::Service* service, // defined `option (idl_support) = true' or not. const bool is_idl_support = sd->file()->options().GetExtension(idl_support); +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (int i = 0; i < sd->method_count(); ++i) { + const auto* md = sd->method(i); + if (_flatbuffers_services->FindMethodByFullName(md->full_name()) || + (is_idl_support && sd->name() != sd->full_name() && + _flatbuffers_services->FindMethodByFullName( + butil::EnsureString(sd->name()) + "." + + butil::EnsureString(md->name())))) { + LOG(ERROR) << "Protobuf method conflicts with FlatBuffers method: " + << md->full_name(); + return -1; + } + } + } +#endif Tabbed* tabbed = dynamic_cast(service); for (int i = 0; i < sd->method_count(); ++i) { @@ -1656,6 +1762,125 @@ int Server::AddService(google::protobuf::Service* service, return AddServiceInternal(service, false, options); } +#if BRPC_WITH_FLATBUFFERS +int Server::AddFlatBuffersService(flatbuffers::Service* service, + ServiceOwnership ownership) { + if (service == nullptr || + (ownership != SERVER_OWNS_SERVICE && + ownership != SERVER_DOESNT_OWN_SERVICE)) { + LOG(ERROR) << "Invalid FlatBuffers service or ownership"; + return -1; + } + if (InitializeOnce() != 0 || status() != READY) { + LOG(ERROR) << "Can't add FlatBuffers service to Server[" << version() + << "] which is " << status_str(status()); + return -1; + } + const flatbuffers::ServiceDescriptor* sd = service->GetDescriptor(); + if (sd == nullptr || sd->name().empty() || sd->full_name().empty() || + sd->method_count() <= 0) { + LOG(ERROR) << "Invalid FlatBuffers service descriptor"; + return -1; + } + if (_flatbuffers_services) { + for (const auto& entry : _flatbuffers_services->services) { + if (entry.second->service == service || entry.first == sd->index() || + entry.second->descriptor->full_name() == sd->full_name()) { + LOG(ERROR) << "Duplicate FlatBuffers service or hash collision: " + << sd->full_name(); + return -1; + } + } + } + + // Keep ownership with the caller until the entire record is published. + auto record = std::make_unique(service, sd); + std::unordered_set method_names; + for (int i = 0; i < sd->method_count(); ++i) { + const flatbuffers::MethodDescriptor* md = sd->method(i); + if (md == nullptr || md->service() != sd || md->index() < 0 || + md->name().empty() || + md->full_name() != sd->full_name() + "." + md->name() || + sd->FindMethodByIndex(md->index()) != md || + !method_names.insert(md->name()).second || + record->methods.count(md->index()) != 0) { + LOG(ERROR) << "Invalid FlatBuffers method in " << sd->full_name(); + return -1; + } + if (_method_map.seek(md->full_name()) != nullptr) { + LOG(ERROR) << "FlatBuffers method conflicts with protobuf method: " + << md->full_name(); + return -1; + } + record->methods.emplace( + md->index(), + std::make_unique(service, md)); + } + if (!_flatbuffers_services) { + _flatbuffers_services = std::make_unique(); + } + auto inserted = _flatbuffers_services->services.emplace( + sd->index(), std::move(record)); + if (!inserted.second) { + return -1; + } + inserted.first->second->ownership = ownership; + return 0; +} + +int Server::RemoveFlatBuffersService(flatbuffers::Service* service) { + if (service == nullptr) { + LOG(ERROR) << "Parameter[service] is NULL"; + return -1; + } + if (InitializeOnce() != 0 || status() != READY) { + LOG(ERROR) << "Can't remove FlatBuffers service from Server[" << version() + << "] which is " << status_str(status()); + return -1; + } + if (_flatbuffers_services) { + for (auto it = _flatbuffers_services->services.begin(); + it != _flatbuffers_services->services.end(); ++it) { + if (it->second->service == service) { + std::unique_ptr removed( + std::move(it->second)); + _flatbuffers_services->services.erase(it); + return 0; + } + } + } + return -1; +} + +size_t Server::GetFlatBuffersServiceCount() const { + return _flatbuffers_services ? _flatbuffers_services->services.size() : 0; +} + +const Server::FlatBuffersMethodProperty* +Server::FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, + int32_t method_index) const { + if (!_flatbuffers_services || method_index < 0) { + return nullptr; + } + auto service = _flatbuffers_services->services.find(service_index); + if (service == _flatbuffers_services->services.end()) { + return nullptr; + } + // Wire IDs are sparse, not declaration-order positions. + const flatbuffers::MethodDescriptor* md = + service->second->descriptor->FindMethodByIndex(method_index); + if (md == nullptr) { + return nullptr; + } + auto method = service->second->methods.find(method_index); + if (method == service->second->methods.end() || + method->second->property.method != md) { + return nullptr; + } + return &method->second->property; +} +#endif + int Server::AddBuiltinService(google::protobuf::Service* service) { ServiceOptions options; options.ownership = SERVER_OWNS_SERVICE; @@ -1775,6 +2000,11 @@ void Server::ClearServices() { << "] which is " << status_str(status()); return; } +#if BRPC_WITH_FLATBUFFERS + // Detach every FlatBuffers method before destroying owned services. + std::unique_ptr removed_flatbuffers_services; + removed_flatbuffers_services.swap(_flatbuffers_services); +#endif for (ServiceMap::const_iterator it = _fullname_service_map.begin(); it != _fullname_service_map.end(); ++it) { if (it->second.ownership == SERVER_OWNS_SERVICE) { @@ -1821,6 +2051,9 @@ void Server::GetStat(ServerStatistics* stat) const { stat->connection_count += _internal_am->ConnectionCount(); } stat->user_service_count = service_count(); +#if BRPC_WITH_FLATBUFFERS + stat->user_service_count += GetFlatBuffersServiceCount(); +#endif stat->builtin_service_count = builtin_service_count(); } @@ -1851,8 +2084,11 @@ void Server::GenerateVersionIfNeeded() { if (!_version.empty()) { return; } - int extra_count = !!_options.nshead_service + !!_options.rtmp_service + + size_t extra_count = !!_options.nshead_service + !!_options.rtmp_service + !!_options.thrift_service + !!_options.redis_service; +#if BRPC_WITH_FLATBUFFERS + extra_count += GetFlatBuffersServiceCount(); +#endif _version.reserve((extra_count + service_count()) * 20); for (ServiceMap::const_iterator it = _fullname_service_map.begin(); it != _fullname_service_map.end(); ++it) { @@ -1863,6 +2099,16 @@ void Server::GenerateVersionIfNeeded() { _version.append(butil::class_name_str(*it->second.service)); } } +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (const auto& service : _flatbuffers_services->services) { + if (!_version.empty()) { + _version.push_back('+'); + } + _version.append(service.second->descriptor->full_name()); + } + } +#endif if (_options.nshead_service) { if (!_version.empty()) { _version.push_back('+'); @@ -2308,6 +2554,15 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ MethodProperty* mp = _method_map.seek(full_method_name); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + return method->max_concurrency; + } + } +#endif break; } return MaxConcurrencyOf(mp); @@ -2320,7 +2575,16 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ } int Server::MaxConcurrencyOf(const butil::StringPiece& full_method_name) const { - return MaxConcurrencyOf(_method_map.seek(full_method_name)); + const MethodProperty* mp = _method_map.seek(full_method_name); +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName(full_method_name); + if (method) { + return method->max_concurrency; + } + } +#endif + return MaxConcurrencyOf(mp); } AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_service_name, @@ -2328,6 +2592,15 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ MethodProperty* mp = const_cast( FindMethodPropertyByFullName(full_service_name, method_name)); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_service_name.as_string() + "." + method_name.as_string()); + if (method) { + return method->max_concurrency; + } + } +#endif LOG(ERROR) << "Fail to find method=" << full_service_name << '/' << method_name; _failed_to_set_max_concurrency_of_method = true; @@ -2338,8 +2611,18 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ int Server::MaxConcurrencyOf(const butil::StringPiece& full_service_name, const butil::StringPiece& method_name) const { - return MaxConcurrencyOf(FindMethodPropertyByFullName( - full_service_name, method_name)); + const MethodProperty* mp = FindMethodPropertyByFullName( + full_service_name, method_name); +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_service_name.as_string() + "." + method_name.as_string()); + if (method) { + return method->max_concurrency; + } + } +#endif + return MaxConcurrencyOf(mp); } AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(google::protobuf::Service* service, @@ -2355,6 +2638,19 @@ int Server::MaxConcurrencyOf(google::protobuf::Service* service, bool& Server::IgnoreEovercrowdedOf(const butil::StringPiece& full_method_name) { MethodProperty* mp = _method_map.seek(full_method_name); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + if (status() != READY) { + LOG(WARNING) << "IgnoreEovercrowdedOf requires a stopped Server"; + return g_default_ignore_eovercrowded; + } + return method->property.ignore_eovercrowded; + } + } +#endif LOG(ERROR) << "Fail to find method=" << full_method_name; _failed_to_set_ignore_eovercrowded = true; return g_default_ignore_eovercrowded; @@ -2379,6 +2675,15 @@ bool Server::IgnoreEovercrowdedOf(const butil::StringPiece& full_method_name) co return g_default_ignore_eovercrowded; } if (mp == nullptr || mp->status == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + return method->property.ignore_eovercrowded; + } + } +#endif return false; } return mp->ignore_eovercrowded; diff --git a/src/brpc/server.h b/src/brpc/server.h index 31dc69af2f..830f0391ca 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -26,6 +26,7 @@ #include "bthread/bthread.h" // Server may need some bthread functions, // e.g. bthread_usleep #include // google::protobuf::Service +#include "butil/config.h" #include "butil/macros.h" // DISALLOW_COPY_AND_ASSIGN #include "butil/containers/doubly_buffered_data.h" // DoublyBufferedData #include "bvar/bvar.h" @@ -49,6 +50,13 @@ namespace brpc { +#if BRPC_WITH_FLATBUFFERS +namespace flatbuffers { +class Service; +class MethodDescriptor; +} // namespace flatbuffers +#endif + class Acceptor; class MethodStatus; class NsheadService; @@ -450,6 +458,15 @@ class Server { }; typedef butil::FlatMap MethodMap; +#if BRPC_WITH_FLATBUFFERS + struct FlatBuffersMethodProperty { + flatbuffers::Service* service; + const flatbuffers::MethodDescriptor* method; + MethodStatus* status; + bool ignore_eovercrowded; + }; +#endif + struct ThreadLocalOptions { bthread_key_t tls_key; const DataFactory* thread_local_data_factory; @@ -521,6 +538,20 @@ class Server { // Returns 0 on success, -1 otherwise. int RemoveService(google::protobuf::Service* service); +#if BRPC_WITH_FLATBUFFERS + // FlatBuffers services are separate from the protobuf service maps. + // The service and its descriptor must remain valid until removal. + // Add/Remove require a stopped server (READY). Ownership transfers only + // after a successful Add; Remove deletes a server-owned service. + // Like AddService/RemoveService, these methods are not thread-safe. + // Full method names must not conflict with registered protobuf methods. + // Returns 0 on success, -1 otherwise. + int AddFlatBuffersService(flatbuffers::Service* service, + ServiceOwnership ownership); + int RemoveFlatBuffersService(flatbuffers::Service* service); + size_t GetFlatBuffersServiceCount() const; +#endif + // Remove all services from this server. // NOTE: clearing services when server is running is forbidden. void ClearServices(); @@ -612,6 +643,7 @@ class Server { // server.MaxConcurrencyOf("example.EchoService.Echo") = 10; // or server.MaxConcurrencyOf("example.EchoService", "Echo") = 10; // or server.MaxConcurrencyOf(&service, "Echo") = 10; + // The string-based forms also support registered FlatBuffers methods. // Note: These interfaces can ONLY be called before the server is started. // And you should NOT set the max_concurrency when you are going to choose // an auto concurrency limiter, eg `options.max_concurrency = "auto"`.If you @@ -712,6 +744,11 @@ friend struct RevertServerStatus; FindMethodPropertyByNameAndIndex(const butil::StringPiece& service_name, int method_index) const; +#if BRPC_WITH_FLATBUFFERS + const FlatBuffersMethodProperty* FindFlatBuffersMethodPropertyByIndex( + uint32_t service_index, int32_t method_index) const; +#endif + const ServiceProperty* FindServicePropertyByFullName(const butil::StringPiece& fullname) const; @@ -782,6 +819,11 @@ friend struct RevertServerStatus; Acceptor* _am; Acceptor* _internal_am; +#if BRPC_WITH_FLATBUFFERS + struct FlatBuffersServiceMap; + std::unique_ptr _flatbuffers_services; +#endif + // Use method->full_name() as key MethodMap _method_map; diff --git a/test/BUILD.bazel b/test/BUILD.bazel index a2c291eb7e..5eec8e014c 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -264,11 +264,30 @@ cc_test( ], ) +cc_test( + name = "brpc_flatbuffers_protocol_unittest", + srcs = [ + "brpc_flatbuffers_protocol_unittest.cpp", + ":flatbuffers_test_messages", + ], + copts = COPTS, + includes = ["."], + tags = ["exclusive"], + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + deps = [ + "//:brpc", + "@com_google_googletest//:gtest_main", + ], +) + generate_unittests( name = "brpc_unittests", srcs = glob( ["brpc_*_unittest.cpp"], - exclude = ["brpc_flatbuffers_unittest.cpp"], + exclude = [ + "brpc_flatbuffers_unittest.cpp", + "brpc_flatbuffers_protocol_unittest.cpp", + ], ), deps = [ ":gperftools_helper", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e2e9a84757..925bf3b1e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -306,19 +306,21 @@ if(WITH_FLATBUFFERS) "${FLATBUFFERS_FLATC_EXECUTABLE}" COMMENT "Generating FlatBuffers test messages" VERBATIM) - add_executable(brpc_flatbuffers_unittest - brpc_flatbuffers_unittest.cpp "${FLATBUFFERS_TEST_HEADER}") - target_include_directories(brpc_flatbuffers_unittest BEFORE PRIVATE - "${BRPC_GTEST_INCLUDE_DIR}") - target_link_libraries(brpc_flatbuffers_unittest PRIVATE - brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) - add_test(NAME brpc_flatbuffers_unittest COMMAND brpc_flatbuffers_unittest) + foreach(FB_TEST brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest) + add_executable(${FB_TEST} ${FB_TEST}.cpp "${FLATBUFFERS_TEST_HEADER}") + target_include_directories(${FB_TEST} BEFORE PRIVATE + "${BRPC_GTEST_INCLUDE_DIR}") + target_link_libraries(${FB_TEST} PRIVATE + brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) + add_test(NAME ${FB_TEST} COMMAND ${FB_TEST}) + endforeach() endif() # brpc tests file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") list(REMOVE_ITEM BRPC_UNITTESTS - "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp") + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_protocol_unittest.cpp") foreach(BRPC_UT ${BRPC_UNITTESTS}) get_filename_component(BRPC_UT_WE ${BRPC_UT} NAME_WE) add_executable(${BRPC_UT_WE} ${BRPC_UT} $) diff --git a/test/Makefile b/test/Makefile index f4d5462e63..3f9103e9fb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -182,7 +182,7 @@ TEST_BTHREAD_OBJS = $(addsuffix .o, $(basename $(TEST_BTHREAD_SOURCES))) TEST_BRPC_SOURCES = $(wildcard brpc_*unittest.cpp) ifneq ($(WITH_FLATBUFFERS),1) -TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp,$(TEST_BRPC_SOURCES)) +TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp brpc_flatbuffers_protocol_unittest.cpp,$(TEST_BRPC_SOURCES)) endif TEST_BRPC_OBJS = $(addsuffix .o, $(basename $(TEST_BRPC_SOURCES))) @@ -248,11 +248,11 @@ $(FLATBUFFERS_TEST_HEADER):flatbuffers_message.fbs @mkdir -p $(FLATBUFFERS_TEST_GEN_DIR) $(FLATC) --cpp -o $(FLATBUFFERS_TEST_GEN_DIR) $< -brpc_flatbuffers_unittest.o:brpc_flatbuffers_unittest.cpp $(FLATBUFFERS_TEST_HEADER) | libbrpc.dbg.$(SOEXT) +brpc_flatbuffers_unittest.o brpc_flatbuffers_protocol_unittest.o:%.o:%.cpp $(FLATBUFFERS_TEST_HEADER) | libbrpc.dbg.$(SOEXT) @echo "> Compiling $@" $(CXX) -c $(FLATBUFFERS_TEST_HDRPATHS) $(CXXFLAGS) $< -o $@ -brpc_flatbuffers_unittest:brpc_flatbuffers_unittest.o | libbrpc.dbg.$(SOEXT) +brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest: %: %.o | libbrpc.dbg.$(SOEXT) @echo "> Linking $@" ifeq ($(SYSTEM),Linux) $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) diff --git a/test/brpc_channel_unittest.cpp b/test/brpc_channel_unittest.cpp index 3663efa6e3..20bd168b1d 100644 --- a/test/brpc_channel_unittest.cpp +++ b/test/brpc_channel_unittest.cpp @@ -334,7 +334,9 @@ class ChannelTest : public ::testing::Test{ nullptr, ProcessRpcRequest, VerifyMyRequest, nullptr, nullptr, brpc::CONNECTION_TYPE_ALL, "baidu_std" }; - ASSERT_EQ(0, RegisterProtocol((brpc::ProtocolType)30, dummy_protocol)); + // Keep the test protocol outside the IDs assigned to builtins. + const auto dummy_type = static_cast(brpc::ProtocolType_MAX + 1); + ASSERT_EQ(0, RegisterProtocol(dummy_type, dummy_protocol)); } static void ProcessRpcRequest(brpc::InputMessageBase* msg_base) { diff --git a/test/brpc_flatbuffers_protocol_unittest.cpp b/test/brpc_flatbuffers_protocol_unittest.cpp new file mode 100644 index 0000000000..e49f5c63f2 --- /dev/null +++ b/test/brpc_flatbuffers_protocol_unittest.cpp @@ -0,0 +1,1922 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "butil/config.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "butil/endpoint.h" +#include "butil/fd_guard.h" +#include "brpc/authenticator.h" +#include "brpc/channel.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/method_status.h" +#include "brpc/details/server_private_accessor.h" +#include "brpc/errno.pb.h" +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include "brpc/interceptor.h" +#include "brpc/retry_policy.h" +#include "brpc/policy/flatbuffers_protocol.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/server.h" +#include "brpc/socket.h" +#include "brpc/stream.h" +#include "flatbuffers_message_generated.h" + +namespace { + +using brpc::flatbuffers::BrpcDescriptorTable; +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::MethodDescriptor; +using brpc::flatbuffers::ServiceDescriptor; +using brpc_fbtest::Payload; + +const int kRpcTimeoutMs = 1500; +const int kWaitTimeoutMs = 5000; +const size_t kHeaderSize = 12; +const size_t kRequestMetaSize = 24; +const size_t kResponseMetaSize = 20; + +// These independent byte-level helpers intentionally do not reuse the codec. +void Put32(std::string* bytes, size_t offset, uint32_t value, + bool big_endian = false) { + for (size_t i = 0; i < 4; ++i) { + (*bytes)[offset + i] = static_cast( + value >> (8 * (big_endian ? 3 - i : i))); + } +} + +uint32_t Get32(const std::string& bytes, size_t offset, + bool big_endian = false) { + uint32_t value = 0; + for (size_t i = 0; i < 4; ++i) { + value |= static_cast( + static_cast(bytes[offset + i])) + << (8 * (big_endian ? 3 - i : i)); + } + return value; +} + +void Put64(std::string* bytes, size_t offset, uint64_t value) { + Put32(bytes, offset, static_cast(value)); + Put32(bytes, offset + 4, static_cast(value >> 32)); +} + +uint64_t Get64(const std::string& bytes, size_t offset) { + return Get32(bytes, offset) | + (static_cast(Get32(bytes, offset + 4)) << 32); +} + +std::string Header(uint32_t body_size, uint32_t meta_size) { + std::string bytes(kHeaderSize, '\0'); + bytes.replace(0, 4, "FRPC"); + Put32(&bytes, 4, body_size, true); + Put32(&bytes, 8, meta_size, true); + return bytes; +} + +std::string Frame(const std::string& meta, const std::string& payload) { + return Header(meta.size() + payload.size(), meta.size()) + meta + payload; +} + +std::string Bytes(const Message& message) { + return std::string(reinterpret_cast(message.data()), + message.size()); +} + +Message MakePayload(size_t text_size = 16, int64_t value = 123, + size_t vector_size = 4) { + MessageBuilder builder(8); + auto text = builder.CreateString(std::string(text_size, 'x')); + std::vector numbers(vector_size); + for (size_t i = 0; i < numbers.size(); ++i) { + numbers[i] = static_cast(i) * -3; + } + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload(builder, value, text, values)); + return builder.ReleaseMessage(); +} + +void ExpectReply(const Message& response, const Message& request, + const MethodDescriptor* method) { + ASSERT_NE(nullptr, method); + ASSERT_TRUE(request.Verify()); + // Framing validation is not schema validation; the caller must do this. + ASSERT_TRUE(response.Verify()); + const Payload* in = request.GetRoot(); + const Payload* out = response.GetRoot(); + EXPECT_EQ(in->value() + method->index(), out->value()); + ASSERT_NE(nullptr, out->message()); + EXPECT_EQ(method->name() + ":" + in->message()->str(), + out->message()->str()); + ASSERT_NE(nullptr, out->values()); + ASSERT_EQ(in->values()->size(), out->values()->size()); + for (size_t i = 0; i < in->values()->size(); ++i) { + EXPECT_EQ(in->values()->Get(i), out->values()->Get(i)); + } +} + +std::string RequestMeta(const MethodDescriptor* method, uint32_t message_size, + uint32_t attachment_size, uint64_t correlation_id) { + std::string meta(kRequestMetaSize, '\0'); + Put32(&meta, 0, method->service()->index()); + Put32(&meta, 4, method->index()); + Put32(&meta, 8, message_size); + Put32(&meta, 12, attachment_size); + Put64(&meta, 16, correlation_id); + return meta; +} + +std::string ResponseMeta(uint32_t message_size, uint32_t attachment_size, + uint64_t correlation_id) { + std::string meta(kResponseMetaSize, '\0'); + Put32(&meta, 4, message_size); + Put32(&meta, 8, attachment_size); + Put64(&meta, 12, correlation_id); + return meta; +} + +class Completion : public google::protobuf::Closure { +public: + void Run() override { + std::lock_guard lock(_mutex); + ++_calls; + _condition.notify_all(); + } + bool Wait() { + std::unique_lock lock(_mutex); + return _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _calls != 0; }); + } + int calls() { + std::lock_guard lock(_mutex); + return _calls; + } +private: + std::mutex _mutex; + std::condition_variable _condition; + int _calls = 0; +}; + +class PayloadService : public brpc::flatbuffers::Service { +public: + explicit PayloadService( + BrpcDescriptorTable table = {"brpc_fbtest", "ProtocolService", + "Echo Alternate Removed", {7, 41, 99}}, + std::atomic* destroyed = nullptr) + : _destroyed(destroyed) { + EXPECT_EQ(0, _descriptor.init(table)); + } + ~PayloadService() override { + ReleaseHeld(); + if (_destroyed) { + ++*_destroyed; + } + } + const ServiceDescriptor* GetDescriptor() override { return &_descriptor; } + void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + brpc::Controller* cntl = static_cast(controller); + ++entered; + if (cntl->is_security_mode()) { + ++security_mode_calls; + } + if (!method || _descriptor.FindMethodByIndex(method->index()) != method || + cntl->flatbuffers_method() != method) { + cntl->SetFailed(brpc::ENOMETHOD, "Wrong FlatBuffers method context"); + return; + } + if (!request || !request->Verify()) { + ++rejected; + cntl->SetFailed(brpc::EREQUEST, "Invalid Payload schema"); + return; + } + // No application field is read before Verify succeeds. + const Payload* in = request->GetRoot(); + if (!in->message() || !in->values()) { + cntl->SetFailed(brpc::EREQUEST, "Missing application fields"); + return; + } + const int failure = fail_next_code.exchange(0); + if (failure != 0) { + cntl->SetFailed(failure, "Requested one-shot application failure"); + return; + } + MessageBuilder builder(8); + auto text = builder.CreateString(method->name() + ":" + + in->message()->str()); + const std::vector numbers(in->values()->begin(), + in->values()->end()); + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload( + builder, in->value() + method->index(), text, values)); + *response = builder.ReleaseMessage(); + cntl->response_attachment().append(cntl->request_attachment()); + if (response_compressed.load()) { + cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP); + } + if (response_checksummed.load()) { + cntl->set_response_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } + std::lock_guard lock(_mutex); + if (_hold_next) { + _hold_next = false; + _held = done_guard.release(); + _condition.notify_all(); + } + } + void HoldNext() { + std::lock_guard lock(_mutex); + _hold_next = true; + } + bool WaitHeld() { + std::unique_lock lock(_mutex); + return _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _held != nullptr; }); + } + void ReleaseHeld() { + google::protobuf::Closure* done = nullptr; + { + std::lock_guard lock(_mutex); + _hold_next = false; + std::swap(done, _held); + } + if (done) { + done->Run(); + } + } + std::atomic entered{0}; + std::atomic rejected{0}; + std::atomic security_mode_calls{0}; + std::atomic fail_next_code{0}; + std::atomic response_compressed{false}; + std::atomic response_checksummed{false}; +private: + ServiceDescriptor _descriptor; + std::atomic* _destroyed; + std::mutex _mutex; + std::condition_variable _condition; + bool _hold_next = false; + google::protobuf::Closure* _held = nullptr; +}; + +int InitChannel(brpc::Channel* channel, const butil::EndPoint& endpoint, + const char* protocol = "fb_rpc", + const char* connection_type = "single") { + brpc::ChannelOptions options; + options.protocol = protocol; + options.connection_type = connection_type; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + return channel->Init(endpoint, &options); +} + +class FlatBuffersProtocolTest : public ::testing::Test { +protected: + void SetUp() override { + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + } + void TearDown() override { + service.ReleaseHeld(); + server.Stop(0); + server.Join(); + } + const MethodDescriptor* method(int id = 7) { + return service.GetDescriptor()->FindMethodByIndex(id); + } + PayloadService service; + brpc::Server server; + brpc::Channel channel; +}; + +TEST(FlatBuffersFramingTest, PartialFramesAndInvalidHeadersDoNotConsumeInput) { + const std::string wire = Frame(std::string(24, 'm'), "payload"); + for (size_t size = 0; size < wire.size(); ++size) { + SCOPED_TRACE(size); + butil::IOBuf input; + input.append(wire.data(), size); + const std::string before = input.to_string(); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + EXPECT_EQ(brpc::PARSE_ERROR_NOT_ENOUGH_DATA, result.error()); + EXPECT_EQ(before, input.to_string()); + } + for (const std::string& bad : {std::string("X"), std::string("FX"), + std::string("FRX"), std::string("FRPX")}) { + butil::IOBuf input; + input.append(bad); + EXPECT_EQ(brpc::PARSE_ERROR_TRY_OTHERS, + brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr).error()); + EXPECT_EQ(bad, input.to_string()); + } + const std::vector > malformed = { + {Header(std::numeric_limits::max(), 24), + brpc::PARSE_ERROR_TOO_BIG_DATA}, + {Header(0, 1), brpc::PARSE_ERROR_ABSOLUTELY_WRONG}, + {Header(23, 24), brpc::PARSE_ERROR_ABSOLUTELY_WRONG} + }; + for (const auto& entry : malformed) { + butil::IOBuf input; + input.append(entry.first); + EXPECT_EQ(entry.second, brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr).error()); + EXPECT_EQ(entry.first, input.to_string()); + } +} + +TEST(FlatBuffersFramingTest, FragmentedUnalignedAndExtendedMetadata) { + const Message payload = MakePayload(8193); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Framing", "Echo", {41}})); + std::string meta = RequestMeta(descriptor.method(0), payload.size(), 3, + 0x1122334455667788ULL); + meta.append("\x01\x02\x03\x04\x05", 5); + const std::string wire = Frame(meta, Bytes(payload) + "att"); + for (size_t fragment_size : {1u, 7u, 13u, 4093u}) { + SCOPED_TRACE(fragment_size); + butil::IOBuf input; + for (size_t pos = 0; pos < wire.size(); pos += fragment_size) { + const size_t size = std::min(fragment_size, wire.size() - pos); + char* allocation = static_cast(malloc(size + 1)); + ASSERT_NE(nullptr, allocation); + memcpy(allocation + 1, wire.data() + pos, size); + ASSERT_EQ(0, input.append_user_data(allocation + 1, size, + [allocation](void*) { free(allocation); })); + } + ASSERT_GT(input.backing_block_num(), 1u); + input.append("next"); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + ASSERT_EQ(brpc::PARSE_OK, result.error()); + brpc::DestroyingPtr parsed( + static_cast(result.message())); + EXPECT_EQ(meta, parsed->meta.to_string()); + EXPECT_EQ(Bytes(payload) + "att", parsed->payload.to_string()); + EXPECT_EQ("next", input.to_string()); + Message decoded; + butil::IOBuf body; + parsed->payload.cutn(&body, payload.size()); + ASSERT_TRUE(decoded.parse_msg_from_iobuf(body, payload.size(), 0)); + ASSERT_TRUE(decoded.Verify()); + EXPECT_EQ(Bytes(payload), Bytes(decoded)); + } +} + +TEST(FlatBuffersFramingTest, EmptyMetadataIsSafelySeparatedForProcessorValidation) { + for (size_t meta_size : {0u, 1u, 19u, 23u, 24u}) { + butil::IOBuf input; + input.append(Frame(std::string(meta_size, '\xff'), "x")); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + ASSERT_EQ(brpc::PARSE_OK, result.error()); + brpc::DestroyingPtr parsed( + static_cast(result.message())); + EXPECT_EQ(meta_size, parsed->meta.size()); + EXPECT_EQ("x", parsed->payload.to_string()); + EXPECT_TRUE(input.empty()); + } +} + +TEST(FlatBuffersSerializationTest, RejectsNullProtobufEmptyCompressionAndChecksum) { + google::protobuf::DescriptorProto protobuf; + Message empty; + Message valid = MakePayload(); + const google::protobuf::Message* invalid[] = {nullptr, &protobuf, &empty}; + for (const auto* request : invalid) { + brpc::Controller cntl; + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, request); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()); + EXPECT_TRUE(serialized.empty()); + } + for (int option = 0; option < 2; ++option) { + brpc::Controller cntl; + if (option == 0) { + cntl.set_request_compress_type(brpc::COMPRESS_TYPE_GZIP); + } else { + cntl.set_request_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, &valid); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()); + EXPECT_TRUE(serialized.empty()); + } +} + +TEST(FlatBuffersSerializationTest, PayloadOnlyAndIndependentRetryHeadersKeepConstInput) { + Message mutable_request = MakePayload(8192); + memset(mutable_request.mutable_buf_begin(), 0xa5, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const uint8_t* data = request.data(); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Pack", "Echo", {41}})); + brpc::Controller cntl; + brpc::ControllerPrivateAccessor(&cntl).set_flatbuffers_method( + descriptor.method(0)); + cntl.request_attachment().append("\0attachment", 11); + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, &request); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(original, serialized.to_string()); + EXPECT_EQ(request.size(), serialized.size()); + butil::IOBuf first; + brpc::policy::PackFlatBuffersRequest(&first, nullptr, + 0x0102030405060708ULL, nullptr, &cntl, serialized, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const std::string first_snapshot = first.to_string(); + cntl.request_attachment().clear(); + cntl.request_attachment().append("second"); + butil::IOBuf second; + brpc::policy::PackFlatBuffersRequest(&second, nullptr, + 0x8877665544332211ULL, nullptr, &cntl, serialized, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const std::string second_snapshot = second.to_string(); + ASSERT_GE(first_snapshot.size(), kHeaderSize + kRequestMetaSize); + ASSERT_GE(second_snapshot.size(), kHeaderSize + kRequestMetaSize); + EXPECT_EQ("FRPC", first_snapshot.substr(0, 4)); + EXPECT_EQ(first_snapshot.size() - 12, Get32(first_snapshot, 4, true)); + EXPECT_EQ(24u, Get32(first_snapshot, 8, true)); + EXPECT_EQ(descriptor.index(), Get32(first_snapshot, 12)); + EXPECT_EQ(41u, Get32(first_snapshot, 16)); + EXPECT_EQ(request.size(), Get32(first_snapshot, 20)); + EXPECT_EQ(11u, Get32(first_snapshot, 24)); + EXPECT_EQ(0x0102030405060708ULL, Get64(first_snapshot, 28)); + EXPECT_EQ(0x8877665544332211ULL, Get64(second_snapshot, 28)); + EXPECT_EQ(6u, Get32(second_snapshot, 24)); + EXPECT_EQ(first_snapshot, first.to_string()); + EXPECT_EQ(original, serialized.to_string()); + EXPECT_EQ(data, request.data()); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(original, first_snapshot.substr(36, request.size())); + EXPECT_EQ(original, second_snapshot.substr(36, request.size())); +} + +class RejectingAuthenticator : public brpc::Authenticator { +public: + int GenerateCredential(std::string* credential) const override { + ++generated; + *credential = "test-credential"; + return 0; + } + int VerifyCredential(const std::string&, const butil::EndPoint&, + brpc::AuthContext*) const override { + return -1; + } + mutable std::atomic generated{0}; +}; + +TEST(FlatBuffersSerializationTest, PackRejectsMissingMethodEmptyPayloadAndAuth) { + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "PackErrors", "Echo", {7}})); + const Message request = MakePayload(); + RejectingAuthenticator auth; + for (int variant = 0; variant < 3; ++variant) { + brpc::Controller cntl; + if (variant != 0) { + brpc::ControllerPrivateAccessor(&cntl).set_flatbuffers_method( + descriptor.method(0)); + } + butil::IOBuf serialized; + if (variant != 1) { + serialized.append(Bytes(request)); + } + butil::IOBuf packed; + brpc::policy::PackFlatBuffersRequest(&packed, nullptr, 1, nullptr, + &cntl, serialized, variant == 2 ? &auth : nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_TRUE(packed.empty()); + } +} + +TEST_F(FlatBuffersProtocolTest, SynchronousAndAsynchronousPayloadAttachmentVariants) { + EXPECT_EQ(30, static_cast(brpc::PROTOCOL_FLATBUFFERS_RPC)); + for (size_t size : {0u, 1u, 63u, 1024u, 8193u, 65537u}) { + for (bool asynchronous : {false, true}) { + SCOPED_TRACE(size); + SCOPED_TRACE(asynchronous); + const Message request = MakePayload(size, size + 31, size % 19); + Message response; + brpc::Controller cntl; + std::string attachment(size / 3, '\0'); + for (size_t i = 0; i < attachment.size(); ++i) { + attachment[i] = static_cast(i % 251); + } + cntl.request_attachment().append(attachment); + Completion done; + channel.FBCallMethod(method(41), &cntl, &request, &response, + asynchronous ? &done : nullptr); + if (asynchronous) { + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + } + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(method(41), cntl.flatbuffers_method()); + EXPECT_EQ(brpc::PROTOCOL_FLATBUFFERS_RPC, cntl.request_protocol()); + ExpectReply(response, request, method(41)); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + } + } +} + +TEST_F(FlatBuffersProtocolTest, ConcurrentCallsShareOneConstMessageWithoutMutation) { + Message mutable_request = MakePayload(16385, 9981, 256); + memset(mutable_request.mutable_buf_begin(), 0x3d, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const MethodDescriptor* echo = method(); + const MethodDescriptor* alternate = method(41); + std::vector clients; + for (int client = 0; client < 6; ++client) { + clients.emplace_back([&, client] { + for (int call = 0; call < 8; ++call) { + brpc::Controller cntl; + Message response; + const std::string attachment = std::to_string(client) + ":" + + std::to_string(call); + cntl.request_attachment().append(attachment); + const MethodDescriptor* selected = call % 2 ? echo : alternate; + channel.FBCallMethod(selected, &cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + if (!cntl.Failed()) { + ExpectReply(response, request, selected); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + } + } + }); + } + for (auto& client : clients) { + client.join(); + } + EXPECT_EQ(48, service.entered.load()); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); +} + +TEST_F(FlatBuffersProtocolTest, UnknownServiceAndMethodDoNotDispatch) { + ServiceDescriptor unknown_service; + ServiceDescriptor unknown_method; + ASSERT_EQ(0, unknown_service.init( + {"brpc_fbtest", "MissingService", "Echo", {7}})); + ASSERT_EQ(0, unknown_method.init( + {"brpc_fbtest", "ProtocolService", "Missing", {12345}})); + const Message request = MakePayload(); + for (const MethodDescriptor* missing : {unknown_service.method(0), + unknown_method.method(0)}) { + Message response; + brpc::Controller cntl; + channel.FBCallMethod(missing, &cntl, &request, &response, nullptr); + EXPECT_EQ(brpc::ENOMETHOD, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } + EXPECT_EQ(0, service.entered.load()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); +} + +TEST_F(FlatBuffersProtocolTest, ServiceVerifiesInvalidPayloadAndRemainsUsable) { + for (size_t size : {1u, 3u, 4u, 16u, 65u}) { + butil::IOBuf bytes; + bytes.append(std::string(size, '\xff')); + Message invalid; + ASSERT_TRUE(invalid.parse_msg_from_iobuf(bytes, size, 0)); + ASSERT_FALSE(invalid.Verify()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &invalid, &response, nullptr); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } + EXPECT_EQ(5, service.rejected.load()); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); + EXPECT_EQ(6, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, AsynchronousValidationErrorsRunCallbackOnce) { + const Message request = MakePayload(); + brpc::Channel protobuf_channel; + ASSERT_EQ(0, InitChannel(&protobuf_channel, server.listen_address(), + "baidu_std")); + for (int variant = 0; variant < 4; ++variant) { + SCOPED_TRACE(variant); + Message response; + brpc::Controller cntl; + Completion done; + brpc::Channel* selected = variant == 0 ? &protobuf_channel : &channel; + selected->FBCallMethod(variant == 1 ? nullptr : method(), &cntl, + variant == 2 ? nullptr : &request, + variant == 3 ? nullptr : &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + } + EXPECT_EQ(0, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, RejectsProtobufOnLegacyCallbackWithoutDispatch) { + google::protobuf::DescriptorProto request; + google::protobuf::DescriptorProto response; + request.set_name("not-a-flatbuffer"); + brpc::Controller cntl; + Completion done; + channel.CallMethod(nullptr, &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, UnsupportedCompressionChecksumAndStreamsFailRpc) { + const Message request = MakePayload(); + for (int variant = 0; variant < 3; ++variant) { + brpc::Controller cntl; + Message response; + brpc::StreamId stream = brpc::INVALID_STREAM_ID; + if (variant == 0) { + cntl.set_request_compress_type(brpc::COMPRESS_TYPE_GZIP); + } else if (variant == 1) { + cntl.set_request_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } else { + ASSERT_EQ(0, brpc::StreamCreate(&stream, cntl, nullptr)); + } + Completion done; + channel.FBCallMethod(method(), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + if (stream != brpc::INVALID_STREAM_ID) { + brpc::StreamClose(stream); + } + } + EXPECT_EQ(0, service.entered.load()); + for (int variant = 0; variant < 2; ++variant) { + service.response_compressed = variant == 0; + service.response_checksummed = variant == 1; + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + EXPECT_EQ(brpc::ERESPONSE, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } +} + +TEST_F(FlatBuffersProtocolTest, ClientAuthenticationIsRejected) { + RejectingAuthenticator auth; + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.auth = &auth; + options.timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + brpc::Channel authenticated; + ASSERT_EQ(0, authenticated.Init(server.listen_address(), &options)); + const Message request = MakePayload(); + brpc::Controller cntl; + Message response; + authenticated.FBCallMethod(method(), &cntl, &request, &response, nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); +} + +TEST(FlatBuffersLifecycleTest, AuthenticationOnServerCannotBeBypassed) { + PayloadService service; + RejectingAuthenticator auth; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.auth = &auth; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(service.GetDescriptor()->method(0), &cntl, &request, + &response, nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0, server.Stop(0)); + EXPECT_EQ(0, server.Join()); +} + +TEST(FlatBuffersLifecycleTest, StableIdsSurviveReorderRemovalAndRestart) { + PayloadService original; + PayloadService reordered({"brpc_fbtest", "ProtocolService", + "Alternate Echo", {41, 7}}); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &original, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + const ServiceDescriptor* client_descriptor = original.GetDescriptor(); + const Message request = MakePayload(); + for (int version = 0; version < 2; ++version) { + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + for (int id : {7, 41, 99}) { + SCOPED_TRACE(version); + SCOPED_TRACE(id); + const MethodDescriptor* method = client_descriptor->FindMethodByIndex(id); + ASSERT_NE(nullptr, method); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method, &cntl, &request, &response, nullptr); + if (version == 1 && id == 99) { + EXPECT_EQ(brpc::ENOMETHOD, cntl.ErrorCode()); + } else { + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method); + } + } + ASSERT_EQ(0, server.Stop(0)); + ASSERT_EQ(0, server.Join()); + if (version == 0) { + ASSERT_EQ(0, server.RemoveFlatBuffersService(&original)); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &reordered, brpc::SERVER_DOESNT_OWN_SERVICE)); + EXPECT_EQ(1, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + } + } + EXPECT_EQ(3, original.entered.load()); + EXPECT_EQ(2, reordered.entered.load()); + server.ClearServices(); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); +} + +class NullDescriptorService : public brpc::flatbuffers::Service { +public: + explicit NullDescriptorService(std::atomic* destroyed) + : _destroyed(destroyed) {} + ~NullDescriptorService() override { ++*_destroyed; } + const ServiceDescriptor* GetDescriptor() override { return nullptr; } + void FBCallMethod(const MethodDescriptor*, + google::protobuf::RpcController* controller, + const Message*, Message*, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + controller->SetFailed("Invalid service was dispatched"); + ADD_FAILURE() << "Invalid service was dispatched"; + } +private: + std::atomic* _destroyed; +}; + +TEST(FlatBuffersLifecycleTest, OwnershipFailureWrongInstanceRemoveAndClear) { + std::atomic owned_destroyed{0}; + std::atomic duplicate_destroyed{0}; + std::atomic invalid_destroyed{0}; + std::atomic borrowed_destroyed{0}; + { + PayloadService borrowed({"brpc_fbtest", "Borrowed", "Echo", {7}}, + &borrowed_destroyed); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &borrowed, brpc::SERVER_DOESNT_OWN_SERVICE)); + std::unique_ptr owned(new PayloadService( + {"brpc_fbtest", "Owned", "Echo", {41}}, &owned_destroyed)); + ASSERT_EQ(0, server.AddFlatBuffersService( + owned.get(), brpc::SERVER_OWNS_SERVICE)); + owned.release(); + EXPECT_EQ(2, server.GetFlatBuffersServiceCount()); + for (brpc::ServiceOwnership ownership : {brpc::SERVER_OWNS_SERVICE, + brpc::SERVER_DOESNT_OWN_SERVICE}) { + std::unique_ptr duplicate(new PayloadService( + {"brpc_fbtest", "Owned", "Echo", {41}}, &duplicate_destroyed)); + const int before = duplicate_destroyed.load(); + EXPECT_NE(0, server.AddFlatBuffersService(duplicate.get(), ownership)); + EXPECT_EQ(before, duplicate_destroyed.load()); + EXPECT_NE(0, server.RemoveFlatBuffersService(duplicate.get())); + EXPECT_EQ(2, server.GetFlatBuffersServiceCount()); + std::unique_ptr invalid( + new NullDescriptorService(&invalid_destroyed)); + const int invalid_before = invalid_destroyed.load(); + EXPECT_NE(0, server.AddFlatBuffersService(invalid.get(), ownership)); + EXPECT_EQ(invalid_before, invalid_destroyed.load()); + } + EXPECT_EQ(2, duplicate_destroyed.load()); + EXPECT_EQ(2, invalid_destroyed.load()); + EXPECT_NE(0, server.AddFlatBuffersService(nullptr, brpc::SERVER_OWNS_SERVICE)); + EXPECT_NE(0, server.RemoveFlatBuffersService(nullptr)); + ASSERT_EQ(0, server.RemoveFlatBuffersService(&borrowed)); + EXPECT_EQ(0, borrowed_destroyed.load()); + EXPECT_EQ(1, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &borrowed, brpc::SERVER_DOESNT_OWN_SERVICE)); + server.ClearServices(); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); + EXPECT_EQ(1, owned_destroyed.load()); + EXPECT_EQ(0, borrowed_destroyed.load()); + server.ClearServices(); + EXPECT_EQ(1, owned_destroyed.load()); + } + EXPECT_EQ(1, borrowed_destroyed.load()); +} + +bool WaitForServerConcurrency(const brpc::Server& server, int expected) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + do { + if (server.Concurrency() == expected) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < deadline); + return server.Concurrency() == expected; +} + +class RejectAllInterceptor : public brpc::Interceptor { +public: + bool Accept(const brpc::Controller* cntl, int& error_code, + std::string& error_text) const override { + ++calls; + EXPECT_EQ(brpc::PROTOCOL_FLATBUFFERS_RPC, cntl->request_protocol()); + EXPECT_NE(nullptr, cntl->flatbuffers_method()); + error_code = EACCES; + error_text = "Rejected by test interceptor"; + return false; + } + mutable std::atomic calls{0}; +}; + +TEST(FlatBuffersLifecycleTest, InterceptorRejectsBeforeServiceAndReleasesConcurrency) { + PayloadService service; + RejectAllInterceptor interceptor; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.interceptor = &interceptor; + options.max_concurrency = 1; + options.method_max_concurrency = 1; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + for (int call = 0; call < 3; ++call) { + brpc::Controller cntl; + Message response; + channel.FBCallMethod(service.GetDescriptor()->method(0), &cntl, + &request, &response, nullptr); + EXPECT_EQ(EACCES, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(call + 1, interceptor.calls.load()); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0u, response.size()); + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + } +} + +TEST(FlatBuffersLifecycleTest, PublicControllerSecurityModeMatchesServerOptions) { + for (bool security_mode : {false, true}) { + SCOPED_TRACE(security_mode); + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.has_builtin_services = !security_mode; + ASSERT_EQ(security_mode, options.security_mode()); + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + brpc::Controller cntl; + Message response; + const MethodDescriptor* method = service.GetDescriptor()->method(0); + channel.FBCallMethod(method, &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method); + EXPECT_EQ(1, service.entered.load()); + EXPECT_EQ(security_mode ? 1 : 0, service.security_mode_calls.load()); + } +} + +int StartWithEphemeralInternalPort(brpc::Server* server, + brpc::ServerOptions* options) { + for (int attempt = 0; attempt < 10; ++attempt) { + butil::EndPoint endpoint; + if (butil::str2endpoint("127.0.0.1:0", &endpoint) != 0) { + return -1; + } + butil::fd_guard reserved(butil::tcp_listen(endpoint)); + if (reserved < 0 || butil::get_local_side(reserved, &endpoint) != 0) { + continue; + } + options->internal_port = endpoint.port; + // internal_port does not support 0. Retry if its ephemeral reservation + // is taken between releasing it and starting the two server listeners. + reserved.reset(-1); + if (server->Start("127.0.0.1:0", options) == 0) { + return 0; + } + } + return -1; +} + +TEST(FlatBuffersLifecycleTest, InternalPortRejectsOrdinaryFlatBuffersServices) { + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + ASSERT_EQ(0, StartWithEphemeralInternalPort(&server, &options)); + ASSERT_TRUE(server.options().security_mode()); + const butil::EndPoint internal_endpoint(server.listen_address().ip, + options.internal_port); + brpc::Channel internal_channel; + brpc::Channel public_channel; + ASSERT_EQ(0, InitChannel(&internal_channel, internal_endpoint)); + ASSERT_EQ(0, InitChannel(&public_channel, server.listen_address())); + const Message request = MakePayload(); + const MethodDescriptor* method = service.GetDescriptor()->method(0); + brpc::Controller rejected; + Message rejected_response; + internal_channel.FBCallMethod(method, &rejected, &request, + &rejected_response, nullptr); + EXPECT_EQ(EPERM, rejected.ErrorCode()) << rejected.ErrorText(); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0u, rejected_response.size()); + + brpc::Controller accepted; + Message response; + public_channel.FBCallMethod(method, &accepted, &request, &response, nullptr); + ASSERT_FALSE(accepted.Failed()) << accepted.ErrorText(); + ExpectReply(response, request, method); + EXPECT_EQ(1, service.entered.load()); + EXPECT_EQ(1, service.security_mode_calls.load()); +} + +void ExerciseMethodConcurrency(bool specific_limit) { + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + const MethodDescriptor* echo = service.GetDescriptor()->FindMethodByIndex(7); + if (specific_limit) { + server.MaxConcurrencyOf(echo->full_name()) = 1; + } + brpc::ServerOptions options; + options.max_concurrency = 8; + options.method_max_concurrency = specific_limit ? 2 : 1; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + for (int id : {7, 41}) { + SCOPED_TRACE(id); + const MethodDescriptor* method = service.GetDescriptor()->FindMethodByIndex(id); + const int limit = specific_limit && id == 41 ? 2 : 1; + const brpc::Server& const_server = server; + EXPECT_EQ(specific_limit && id == 7 ? 1 : 0, + const_server.MaxConcurrencyOf(method->full_name())); + const auto* property = brpc::ServerPrivateAccessor(&server). + FindFlatBuffersMethodPropertyByIndex(method->service()->index(), id); + ASSERT_NE(nullptr, property); + ASSERT_NE(nullptr, property->status); + EXPECT_EQ(limit, property->status->MaxConcurrency()); + const int entered_before = service.entered.load(); + service.HoldNext(); + brpc::Controller held_cntl; + held_cntl.set_timeout_ms(kWaitTimeoutMs); + Message held_response; + Completion held_done; + channel.FBCallMethod(method, &held_cntl, &request, &held_response, &held_done); + const bool held = service.WaitHeld(); + EXPECT_TRUE(held); + // No fatal assertions while a service completion is held: all exits + // below must release it before the server's destructor calls Join(). + if (held) { + EXPECT_TRUE(WaitForServerConcurrency(server, 1)); + for (int attempt = 0; attempt < 2; ++attempt) { + brpc::Controller contender; + Message contender_response; + channel.FBCallMethod(method, &contender, &request, + &contender_response, nullptr); + if (limit == 1) { + EXPECT_EQ(brpc::ELIMIT, contender.ErrorCode()) + << contender.ErrorText(); + EXPECT_EQ(entered_before + 1, service.entered.load()); + EXPECT_EQ(0u, contender_response.size()); + } else { + EXPECT_FALSE(contender.Failed()) << contender.ErrorText(); + if (!contender.Failed()) { + ExpectReply(contender_response, request, method); + } + EXPECT_EQ(entered_before + attempt + 2, service.entered.load()); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 1)); + } + } + service.ReleaseHeld(); + const bool completed = held_done.Wait(); + EXPECT_TRUE(completed); + EXPECT_EQ(1, held_done.calls()); + EXPECT_FALSE(held_cntl.Failed()) << held_cntl.ErrorText(); + if (completed && !held_cntl.Failed()) { + ExpectReply(held_response, request, method); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + brpc::Controller recovered; + Message recovered_response; + channel.FBCallMethod(method, &recovered, &request, &recovered_response, nullptr); + EXPECT_FALSE(recovered.Failed()) << recovered.ErrorText(); + if (!recovered.Failed()) { + ExpectReply(recovered_response, request, method); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + } +} + +TEST(FlatBuffersLifecycleTest, DefaultMethodConcurrencyRecoversAfterRejection) { + ExerciseMethodConcurrency(false); +} + +TEST(FlatBuffersLifecycleTest, SpecificMethodConcurrencyOverridesDefaultAndRecovers) { + ExerciseMethodConcurrency(true); +} + +class RetryOnlyAccessDenied : public brpc::RetryPolicy { +public: + bool DoRetry(const brpc::Controller* cntl) const override { + if (cntl->ErrorCode() == EACCES) { + ++retries; + return true; + } + return false; + } + mutable std::atomic retries{0}; +}; + +TEST_F(FlatBuffersProtocolTest, RealNetworkRetryPreservesConstRequestAndAttachment) { + RetryOnlyAccessDenied policy; + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.connection_type = "single"; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 2; + options.retry_policy = &policy; + brpc::Channel retry_channel; + ASSERT_EQ(0, retry_channel.Init(server.listen_address(), &options)); + Message mutable_request = MakePayload(8193, 7654, 19); + memset(mutable_request.mutable_buf_begin(), 0x69, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const std::string attachment("\0retry\xff", 7); + service.fail_next_code = EACCES; + brpc::Controller cntl; + cntl.request_attachment().append(attachment); + Message response; + Completion done; + retry_channel.FBCallMethod(method(41), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(1, done.calls()); + EXPECT_EQ(1, cntl.retried_count()); + EXPECT_EQ(1, policy.retries.load()); + EXPECT_EQ(2, service.entered.load()); + ExpectReply(response, request, method(41)); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); + + // The same policy must not retry a different application error. + service.fail_next_code = brpc::EREQUEST; + brpc::Controller no_retry; + Message no_retry_response; + retry_channel.FBCallMethod(method(), &no_retry, &request, + &no_retry_response, nullptr); + EXPECT_EQ(brpc::EREQUEST, no_retry.ErrorCode()); + EXPECT_EQ(0, no_retry.retried_count()); + EXPECT_EQ(1, policy.retries.load()); + EXPECT_EQ(3, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, PooledAndShortConnectionsSupportSyncAndAsyncCalls) { + for (const char* connection_type : {"pooled", "short"}) { + SCOPED_TRACE(connection_type); + brpc::Channel selected; + ASSERT_EQ(0, InitChannel(&selected, server.listen_address(), + "fb_rpc", connection_type)); + for (size_t size : {0u, 8193u}) { + for (bool asynchronous : {false, true}) { + const Message request = MakePayload(size, size + 23, 7); + brpc::Controller cntl; + cntl.request_attachment().append("connection-attachment"); + Message response; + Completion done; + selected.FBCallMethod(method(), &cntl, &request, &response, + asynchronous ? &done : nullptr); + if (asynchronous) { + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + } + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const brpc::ConnectionType expected_type = + std::string(connection_type) == "pooled" ? + brpc::CONNECTION_TYPE_POOLED : brpc::CONNECTION_TYPE_SHORT; + EXPECT_EQ(expected_type, cntl.connection_type()); + ExpectReply(response, request, method()); + EXPECT_EQ("connection-attachment", + cntl.response_attachment().to_string()); + } + } + } + EXPECT_EQ(8, service.entered.load()); +} + +struct DeferredJoinState { + ~DeferredJoinState() { service.ReleaseHeld(); } + PayloadService service; + Message request = MakePayload(); + Message response; + brpc::Controller cntl; + Completion done; + // Destroy the server before the request/controller/service on early exits. + brpc::Server server; + std::mutex mutex; + std::condition_variable condition; + bool join_started = false; + bool joined = false; + int join_result = -1; +}; + +TEST(FlatBuffersLifecycleTest, StopRejectsRemovalUntilDeferredDoneAndJoin) { + auto state = std::make_shared(); + ASSERT_EQ(0, state->server.AddFlatBuffersService( + &state->service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, state->server.Start("127.0.0.1:0", nullptr)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, state->server.listen_address())); + const MethodDescriptor* method = state->service.GetDescriptor()->method(0); + state->service.HoldNext(); + state->cntl.set_timeout_ms(kWaitTimeoutMs); + channel.FBCallMethod(method, &state->cntl, &state->request, + &state->response, &state->done); + EXPECT_TRUE(state->service.WaitHeld()); + EXPECT_EQ(0, state->done.calls()); + EXPECT_NE(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(0, state->server.Stop(0)); + EXPECT_NE(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(1, state->server.GetFlatBuffersServiceCount()); + + std::thread joiner([state] { + { + std::lock_guard lock(state->mutex); + state->join_started = true; + state->condition.notify_all(); + } + const int result = state->server.Join(); + { + std::lock_guard lock(state->mutex); + state->join_result = result; + state->joined = true; + state->condition.notify_all(); + } + }); + { + std::unique_lock lock(state->mutex); + EXPECT_TRUE(state->condition.wait_for( + lock, std::chrono::milliseconds(kWaitTimeoutMs), + [&state] { return state->join_started; })); + EXPECT_FALSE(state->condition.wait_for( + lock, std::chrono::milliseconds(100), + [&state] { return state->joined; })); + } + EXPECT_EQ(0, state->done.calls()); + state->service.ReleaseHeld(); + const bool completed = state->done.Wait(); + EXPECT_TRUE(completed); + EXPECT_EQ(1, state->done.calls()); + EXPECT_FALSE(state->cntl.Failed()) << state->cntl.ErrorText(); + if (completed && !state->cntl.Failed()) { + ExpectReply(state->response, state->request, method); + } + bool joined; + { + std::unique_lock lock(state->mutex); + joined = state->condition.wait_for( + lock, std::chrono::milliseconds(kWaitTimeoutMs), + [&state] { return state->joined; }); + } + if (!joined) { + // A broken Join must fail rather than hang the suite. The thread owns + // every object it may still access, so detaching cannot use dead locals. + joiner.detach(); + FAIL() << "Server::Join did not return after deferred done completed"; + } + joiner.join(); + EXPECT_EQ(0, state->join_result); + ASSERT_EQ(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(0, state->server.GetFlatBuffersServiceCount()); +} + +// A normal protobuf service built from descriptors avoids another generated +// test proto while exercising the actual baidu_std serializer and dispatcher. +class DynamicEchoService : public google::protobuf::Service { +public: + DynamicEchoService() { + google::protobuf::FileDescriptorProto file; + file.set_name("flatbuffers_protocol_pb_compat.proto"); + file.set_package("brpc_fbtest_pb"); + auto* message = file.add_message_type(); + message->set_name("Payload"); + auto* field = message->add_field(); + field->set_name("text"); + field->set_number(1); + field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING); + field->set_label(google::protobuf::FieldDescriptorProto::LABEL_OPTIONAL); + auto* service = file.add_service(); + service->set_name("EchoService"); + auto* method = service->add_method(); + method->set_name("Echo"); + method->set_input_type(".brpc_fbtest_pb.Payload"); + method->set_output_type(".brpc_fbtest_pb.Payload"); + const auto* built = _pool.BuildFile(file); + if (built) { + _descriptor = built->service(0); + _prototype = _factory.GetPrototype(built->message_type(0)); + } + } + const google::protobuf::ServiceDescriptor* GetDescriptor() override { + return _descriptor; + } + const google::protobuf::Message& GetRequestPrototype( + const google::protobuf::MethodDescriptor*) const override { + return *_prototype; + } + const google::protobuf::Message& GetResponsePrototype( + const google::protobuf::MethodDescriptor*) const override { + return *_prototype; + } + void CallMethod(const google::protobuf::MethodDescriptor*, + google::protobuf::RpcController* controller, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + auto* cntl = static_cast(controller); + EXPECT_EQ(nullptr, cntl->flatbuffers_method()); + response->CopyFrom(*request); + } +private: + google::protobuf::DescriptorPool _pool; + google::protobuf::DynamicMessageFactory _factory; + const google::protobuf::ServiceDescriptor* _descriptor = nullptr; + const google::protobuf::Message* _prototype = nullptr; +}; + +TEST(FlatBuffersLifecycleTest, ControllerResetClearsContextBeforeOrdinaryProtobufRpc) { + PayloadService flatbuffers_service; + DynamicEchoService protobuf_service; + ASSERT_NE(nullptr, protobuf_service.GetDescriptor()); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &flatbuffers_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.AddService(&protobuf_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + brpc::Channel fb_channel; + brpc::Channel pb_channel; + ASSERT_EQ(0, InitChannel(&fb_channel, server.listen_address())); + ASSERT_EQ(0, InitChannel(&pb_channel, server.listen_address(), "baidu_std")); + const MethodDescriptor* fb_method = flatbuffers_service.GetDescriptor()->method(0); + const Message fb_request = MakePayload(); + Message fb_response; + brpc::Controller cntl; + fb_channel.FBCallMethod(fb_method, &cntl, &fb_request, &fb_response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(fb_method, cntl.flatbuffers_method()); + cntl.Reset(); + EXPECT_EQ(nullptr, cntl.flatbuffers_method()); + EXPECT_EQ(nullptr, cntl.method()); + const auto* method = protobuf_service.GetDescriptor()->method(0); + std::unique_ptr request( + protobuf_service.GetRequestPrototype(method).New()); + std::unique_ptr response( + protobuf_service.GetResponsePrototype(method).New()); + const auto* field = request->GetDescriptor()->FindFieldByName("text"); + ASSERT_NE(nullptr, field); + request->GetReflection()->SetString(request.get(), field, "ordinary protobuf"); + pb_channel.CallMethod(method, &cntl, request.get(), response.get(), nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(nullptr, cntl.flatbuffers_method()); + EXPECT_EQ(method, cntl.method()); + EXPECT_EQ("ordinary protobuf", + response->GetReflection()->GetString(*response, field)); + EXPECT_EQ(0, server.Stop(0)); + EXPECT_EQ(0, server.Join()); +} + +TEST(FlatBuffersLifecycleTest, AuthenticatedProtobufSocketCannotBypassFlatBuffersAuthCheck) { + PayloadService flatbuffers_service; + DynamicEchoService protobuf_service; + RejectingAuthenticator auth; + brpc::Server server; + ASSERT_NE(nullptr, protobuf_service.GetDescriptor()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &flatbuffers_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.AddService(&protobuf_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + brpc::ChannelOptions options; + options.protocol = "baidu_std"; + options.connection_type = "single"; + options.auth = &auth; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + brpc::Channel pb_channel; + brpc::Channel fb_channel; + ASSERT_EQ(0, pb_channel.Init(server.listen_address(), &options)); + // The socket pool key includes the auth pointer, but not the protocol. + options.protocol = "fb_rpc"; + ASSERT_EQ(0, fb_channel.Init(server.listen_address(), &options)); + const auto* pb_method = protobuf_service.GetDescriptor()->method(0); + std::unique_ptr pb_request( + protobuf_service.GetRequestPrototype(pb_method).New()); + std::unique_ptr pb_response( + protobuf_service.GetResponsePrototype(pb_method).New()); + const auto* field = pb_request->GetDescriptor()->FindFieldByName("text"); + ASSERT_NE(nullptr, field); + pb_request->GetReflection()->SetString(pb_request.get(), field, "prime auth socket"); + brpc::Controller first_pb; + pb_channel.CallMethod(pb_method, &first_pb, pb_request.get(), + pb_response.get(), nullptr); + ASSERT_FALSE(first_pb.Failed()) << first_pb.ErrorText(); + ASSERT_EQ(1, auth.generated.load()); + ASSERT_GT(first_pb.local_side().port, 0); + const butil::EndPoint first_connection = first_pb.local_side(); + brpc::ServerStatistics statistics; + server.GetStat(&statistics); + ASSERT_EQ(1u, statistics.connection_count); + + const Message request = MakePayload(); + Message response; + brpc::Controller rejected; + Completion done; + fb_channel.FBCallMethod(flatbuffers_service.GetDescriptor()->method(0), + &rejected, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(rejected.Failed()) + << "Previously authenticated socket bypassed FRPC auth check"; + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, flatbuffers_service.entered.load()); + EXPECT_EQ(1, auth.generated.load()); + + // Verify the primed connection stayed usable and did not authenticate again. + brpc::Controller second_pb; + pb_response->Clear(); + pb_channel.CallMethod(pb_method, &second_pb, pb_request.get(), + pb_response.get(), nullptr); + ASSERT_FALSE(second_pb.Failed()) << second_pb.ErrorText(); + EXPECT_EQ(first_connection, second_pb.local_side()); + EXPECT_EQ("prime auth socket", + pb_response->GetReflection()->GetString(*pb_response, field)); + EXPECT_EQ(1, auth.generated.load()); + server.GetStat(&statistics); + EXPECT_EQ(1u, statistics.connection_count); +} + +// Raw peers use both an absolute deadline and socket timeouts; a regression +// must fail a test instead of leaving accept/read/write blocked indefinitely. +bool ConfigureSocket(int fd) { + struct timeval timeout = {kRpcTimeoutMs / 1000, + (kRpcTimeoutMs % 1000) * 1000}; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) != 0 || + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)) != 0) { + return false; + } +#ifdef SO_NOSIGPIPE + const int enabled = 1; + if (setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &enabled, sizeof(enabled)) != 0) { + return false; + } +#endif + return true; +} + +bool WaitFd(int fd, short events, + const std::chrono::steady_clock::time_point& deadline) { + while (true) { + const auto remaining = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()).count(); + if (remaining <= 0) { + return false; + } + struct pollfd descriptor = {fd, events, 0}; + const int rc = poll(&descriptor, 1, static_cast(remaining)); + if (rc > 0) { + return (descriptor.revents & (events | POLLHUP | POLLERR)) != 0; + } + if (rc == 0 || errno != EINTR) { + return false; + } + } +} + +bool ReadExactly(int fd, char* bytes, size_t size) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + size_t offset = 0; + while (offset < size && WaitFd(fd, POLLIN, deadline)) { + const ssize_t count = recv(fd, bytes + offset, size - offset, 0); + if (count > 0) { + offset += count; + } else if (count == 0 || errno != EINTR) { + return false; + } + } + return offset == size; +} + +bool WriteExactly(int fd, const std::string& bytes) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + size_t offset = 0; + while (offset < bytes.size() && WaitFd(fd, POLLOUT, deadline)) { +#ifdef MSG_NOSIGNAL + const int flags = MSG_NOSIGNAL; +#else + const int flags = 0; +#endif + const ssize_t count = send(fd, bytes.data() + offset, + bytes.size() - offset, flags); + if (count > 0) { + offset += count; + } else if (count == 0 || errno != EINTR) { + return false; + } + } + return offset == bytes.size(); +} + +bool ReadFrame(int fd, std::string* meta, std::string* payload) { + std::string header(kHeaderSize, '\0'); + if (!ReadExactly(fd, &header[0], header.size()) || + header.compare(0, 4, "FRPC") != 0) { + return false; + } + const uint32_t body_size = Get32(header, 4, true); + const uint32_t meta_size = Get32(header, 8, true); + if (meta_size > body_size || body_size > 1024 * 1024) { + return false; + } + std::string body(body_size, '\0'); + if (body_size && !ReadExactly(fd, &body[0], body.size())) { + return false; + } + *meta = body.substr(0, meta_size); + *payload = body.substr(meta_size); + return true; +} + +TEST_F(FlatBuffersProtocolTest, ExtendedRequestMetadataTraversesRealSocket) { + const Message request = MakePayload(8193, 7123, 17); + const std::string attachment("\0binary\xff", 8); + std::string meta = RequestMeta(method(41), request.size(), attachment.size(), + 0x1122334455667788ULL); + meta.append("unknown future fields", 21); + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + const std::string wire = Frame(meta, Bytes(request) + attachment); + ASSERT_TRUE(WriteExactly(fd, wire.substr(0, 5))); + ASSERT_TRUE(WriteExactly(fd, wire.substr(5, 13))); + ASSERT_TRUE(WriteExactly(fd, wire.substr(18))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + EXPECT_EQ(0u, Get32(reply_meta, 0)); + EXPECT_EQ(0x1122334455667788ULL, Get64(reply_meta, 12)); + const uint32_t message_size = Get32(reply_meta, 4); + ASSERT_EQ(reply_payload.size(), message_size + attachment.size()); + EXPECT_EQ(attachment.size(), Get32(reply_meta, 8)); + EXPECT_EQ(attachment, reply_payload.substr(message_size)); + butil::IOBuf body; + body.append(reply_payload.data(), message_size); + Message response; + ASSERT_TRUE(response.parse_msg_from_iobuf(body, message_size, 0)); + ExpectReply(response, request, method(41)); +} + +TEST_F(FlatBuffersProtocolTest, MalformedRequestLengthsFailBeforeServiceDispatch) { + const Message request = MakePayload(); + const std::string payload = Bytes(request) + "att"; + for (int variant = 0; variant < 6; ++variant) { + SCOPED_TRACE(variant); + std::string meta = RequestMeta(method(), request.size(), 3, 991 + variant); + switch (variant) { + case 0: Put32(&meta, 8, 0); break; + case 1: Put32(&meta, 8, 0xffffffffu); break; + case 2: Put32(&meta, 12, 0xffffffffu); break; + case 3: Put32(&meta, 8, request.size() + 1); break; + case 4: Put32(&meta, 12, 2); break; + case 5: Put32(&meta, 4, 0xffffffffu); break; + } + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + ASSERT_TRUE(WriteExactly(fd, Frame(meta, payload))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + EXPECT_EQ(static_cast(variant == 5 ? brpc::ENOMETHOD : + brpc::EREQUEST), + Get32(reply_meta, 0)); + EXPECT_EQ(static_cast(991 + variant), Get64(reply_meta, 12)); + EXPECT_EQ(0u, Get32(reply_meta, 4)); + EXPECT_EQ(0u, Get32(reply_meta, 8)); + EXPECT_TRUE(reply_payload.empty()); + } + EXPECT_EQ(0, service.entered.load()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); +} + +TEST_F(FlatBuffersProtocolTest, TruncatedMetadataAndImpossibleHeaderCloseConnection) { + std::vector frames; + for (size_t size : {0u, 1u, 8u, 19u, 23u}) { + frames.push_back(Frame(std::string(size, '\xff'), "invalid")); + } + frames.push_back(Header(0, 24)); + const Message warmup_request = MakePayload(); + int warmups = 0; + for (const auto& wire : frames) { + SCOPED_TRACE(wire.size()); + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + // NSHEAD precedes FRPC during initial multi-protocol discovery and + // needs 28 bytes. First establish FRPC on this exact connection. + const uint64_t correlation_id = 9001 + warmups; + ASSERT_TRUE(WriteExactly(fd, Frame( + RequestMeta(method(), warmup_request.size(), 0, correlation_id), + Bytes(warmup_request)))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + ASSERT_EQ(0u, Get32(reply_meta, 0)); + EXPECT_EQ(correlation_id, Get64(reply_meta, 12)); + ASSERT_EQ(reply_payload.size(), Get32(reply_meta, 4)); + EXPECT_EQ(0u, Get32(reply_meta, 8)); + butil::IOBuf reply_bytes; + reply_bytes.append(reply_payload); + Message warmup_response; + ASSERT_TRUE(warmup_response.parse_msg_from_iobuf( + reply_bytes, reply_payload.size(), 0)); + ExpectReply(warmup_response, warmup_request, method()); + EXPECT_EQ(++warmups, service.entered.load()); + + ASSERT_TRUE(WriteExactly(fd, wire)); + char byte; + ssize_t count; + do { + count = recv(fd, &byte, 1, 0); + } while (count < 0 && errno == EINTR); + // A receive timeout is not evidence that the server rejected the frame. + EXPECT_TRUE(count == 0 || (count < 0 && errno == ECONNRESET)) + << "recv=" << count << " errno=" << errno; + EXPECT_EQ(warmups, service.entered.load()); + } + EXPECT_EQ(static_cast(frames.size()), service.entered.load()); +} + +class ScriptedPeer { +public: + using Reply = std::function; + bool Start(Reply reply) { + butil::EndPoint endpoint; + if (butil::str2endpoint("127.0.0.1:0", &endpoint) != 0) { + return false; + } + _listener.reset(butil::tcp_listen(endpoint)); + if (_listener < 0 || butil::get_local_side(_listener, &_endpoint) != 0) { + return false; + } + _thread = std::thread([this, reply] { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + if (!WaitFd(_listener, POLLIN, deadline)) { + return; + } + butil::fd_guard connection(accept(_listener, nullptr, nullptr)); + if (connection < 0 || !ConfigureSocket(connection)) { + return; + } + std::string request_meta; + std::string request_payload; + if (!ReadFrame(connection, &request_meta, &request_payload) || + request_meta.size() < kRequestMetaSize) { + return; + } + _served = WriteExactly(connection, reply(request_meta, request_payload)); + // Do not let EOF accidentally turn an accepted malformed response + // into an RPC failure. Keep the connection open until the assertion. + std::unique_lock lock(_mutex); + _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _release; }); + }); + return true; + } + ~ScriptedPeer() { Join(); } + void Join() { + { + std::lock_guard lock(_mutex); + _release = true; + _condition.notify_all(); + } + if (_thread.joinable()) { + _thread.join(); + } + } + butil::EndPoint endpoint() const { return _endpoint; } + bool served() const { return _served.load(); } +private: + butil::fd_guard _listener; + butil::EndPoint _endpoint; + std::thread _thread; + std::atomic _served{false}; + std::mutex _mutex; + std::condition_variable _condition; + bool _release = false; +}; + +TEST(FlatBuffersResponseTest, ExtendedResponseMetadataUsesDeclaredPayloadOffset) { + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([](const std::string& request_meta, + const std::string& request_payload) { + const uint32_t message_size = Get32(request_meta, 8); + const uint32_t attachment_size = Get32(request_meta, 12); + std::string meta = ResponseMeta(message_size, attachment_size, + Get64(request_meta, 16)); + meta.append("\xff\x01\0\x7f\x03", 5); + return Frame(meta, request_payload); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {7}})); + const Message request = MakePayload(8193); + Message response; + brpc::Controller cntl; + cntl.request_attachment().append("\0tail", 5); + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_TRUE(response.Verify()); + EXPECT_EQ(Bytes(request), Bytes(response)); + EXPECT_EQ(std::string("\0tail", 5), cntl.response_attachment().to_string()); + peer.Join(); + EXPECT_TRUE(peer.served()); +} + +TEST(FlatBuffersResponseTest, MalformedResponseFramesFailRpcNotMerelySchemaVerification) { + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {41}})); + const Message request = MakePayload(); + for (int variant = 0; variant < 12; ++variant) { + SCOPED_TRACE(variant); + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([variant](const std::string& request_meta, + const std::string& request_payload) { + std::string meta = ResponseMeta(Get32(request_meta, 8), 0, + Get64(request_meta, 16)); + std::string payload = request_payload; + switch (variant) { + case 0: meta.clear(); break; + case 1: meta.resize(1); break; + case 2: meta.resize(19); break; + case 3: Put32(&meta, 4, 0xffffffffu); break; + case 4: Put32(&meta, 8, 0xffffffffu); break; + case 5: Put32(&meta, 4, payload.size() + 1); break; + case 6: Put32(&meta, 8, 1); break; + case 7: payload.clear(); Put32(&meta, 4, 0); break; + case 8: return Header(0, 20); + case 9: return Header(0xffffffffu, 20); + case 10: Put64(&meta, 12, Get64(request_meta, 16) + (1ULL << 32)); break; + case 11: { + std::string wire = Frame(meta, payload); + wire.resize(wire.size() - 1); + return wire; + } + } + return Frame(meta, payload); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + Message response; + brpc::Controller cntl; + Completion done; + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()) << "Malformed frame reported RPC success"; + if (variant >= 3 && variant <= 7) { + EXPECT_EQ(brpc::ERESPONSE, cntl.ErrorCode()) << cntl.ErrorText(); + } + EXPECT_EQ(0u, response.size()); + peer.Join(); + EXPECT_TRUE(peer.served()); + } +} + +TEST(FlatBuffersResponseTest, SchemaVerificationIsExplicitlyTheCallersResponsibility) { + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([](const std::string& request_meta, const std::string&) { + const std::string invalid(16, '\xff'); + return Frame(ResponseMeta(invalid.size(), 0, Get64(request_meta, 16)), + invalid); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {7}})); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, nullptr); + // The wire sizes are valid; no typed schema is known by the FRPC transport. + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_FALSE(response.Verify()); + peer.Join(); + EXPECT_TRUE(peer.served()); +} + +// Builds a response frame for `cid' as if it had arrived on `socket'. +brpc::policy::MostCommonMessage* MakeFlatBuffersResponse( + const Message& payload, brpc::CallId cid, brpc::Socket* socket) { + brpc::policy::MostCommonMessage* msg = brpc::policy::MostCommonMessage::Get(); + msg->meta.append(ResponseMeta(payload.size(), 0, cid.value)); + msg->payload.append(Bytes(payload)); + socket->ReAddress(&msg->_socket); + return msg; +} + +brpc::SocketUniquePtr CreateTestSocket() { + brpc::SocketId id; + brpc::SocketUniquePtr ptr; + EXPECT_EQ(0, brpc::Socket::Create(brpc::SocketOptions(), &id)); + EXPECT_EQ(0, brpc::Socket::Address(id, &ptr)); + return ptr; +} + +TEST(FlatBuffersResponseTest, ResponseFromForeignSocketIsDropped) { + const Message expected = MakePayload(64, 7); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 2)); + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + brpc::SocketUniquePtr sending = CreateTestSocket(); + brpc::SocketUniquePtr foreign = CreateTestSocket(); + ASSERT_TRUE(sending && foreign); + sending->ReAddress(&cntl._current_call.sending_sock); + + // A live correlation id on another connection must not complete the RPC. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), foreign.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + // The call stays available for the response on its own socket. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), sending.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + +TEST(FlatBuffersResponseTest, ResponseFromPreviousRetryIsDropped) { + const Message expected = MakePayload(64, 8); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 3)); + const brpc::CallId previous_id = cntl.current_id(); + cntl._current_call.nretry = 1; + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + brpc::SocketUniquePtr sending = CreateTestSocket(); + ASSERT_TRUE(sending); + sending->ReAddress(&cntl._current_call.sending_sock); + + // Even on the right socket, a finished attempt cannot supply the result. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, previous_id, sending.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), sending.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + +TEST(FlatBuffersResponseTest, UnfinishedBackupAttemptMatchesOnlyItsOwnSocket) { + const Message expected = MakePayload(64, 9); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 3)); + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + const brpc::CallId first_id = cntl.current_id(); + brpc::SocketUniquePtr first = CreateTestSocket(); + brpc::SocketUniquePtr backup = CreateTestSocket(); + ASSERT_TRUE(first && backup); + first->ReAddress(&cntl._current_call.sending_sock); + // Mirror starting a backup request: keep the first attempt and its socket. + cntl._unfinished_call = new brpc::Controller::Call(&cntl._current_call); + ++cntl._current_call.nretry; + backup->ReAddress(&cntl._current_call.sending_sock); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, first_id, backup.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, first_id, first.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + +} // namespace +#endif // BRPC_WITH_FLATBUFFERS diff --git a/tools/flatbuffers/README.md b/tools/flatbuffers/README.md index 93bc4f988d..23cda6bb58 100644 --- a/tools/flatbuffers/README.md +++ b/tools/flatbuffers/README.md @@ -1,7 +1,11 @@ # Standalone FlatBuffers service generator -`brpc_flatc` generates the bRPC **service abstraction**, not a network protocol or -an adapter for `brpc::Channel`. It uses the installed, official +For runtime enablement and a complete client/server check, see the +[English guide](../../docs/en/flatbuffers.md) or +[中文指南](../../docs/cn/flatbuffers.md). + +`brpc_flatc` generates bRPC service bindings. The separate `fb_rpc` transport +implements their channel interface in `brpc::Channel`. The generator uses the installed, official `flatbuffers/idl.h` Parser and `libflatbuffers`; it does not download or modify a FlatBuffers fork. The official `flatc --cpp` remains responsible for table types. @@ -20,6 +24,12 @@ flatc --cpp -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs /tmp/brpc-codegen-build/brpc_flatc -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs ``` +For a non-default installation, add `-DCMAKE_PREFIX_PATH=/path/to/flatbuffers` +and/or explicit `FLATBUFFERS_INCLUDE_DIR` and `FLATBUFFERS_LIBRARY` values. +Run official `flatc` from that matching installation, not an unrelated binary +found earlier on `PATH`. The runtime alone does not link `libflatbuffers`, but +this generator needs it. + The output directory must already exist. The command line is: ```text @@ -32,7 +42,8 @@ For `echo.fbs`, the two tools produce: - `brpc_flatc`: `echo.brpc.fb.h` and `echo.brpc.fb.cpp`, both Apache-licensed. Compile the generated `.cpp` into the application with C++14 or newer and bRPC -configured with FlatBuffers support. Keep the generated header, official table header, and bRPC +configured with FlatBuffers support. When the detected `Protobuf_VERSION` is +greater than 4.21, use C++17 and the matching Protobuf/Abseil dependency set. Keep the generated header, official table header, and bRPC headers on the include path. The generated service files do not pin a FlatBuffers version; use an official `flatc` matching your installed table headers, as required by official generated code. @@ -69,8 +80,11 @@ The generated names are `example::api::Echo`, They are **not** the old `EchoStub` spelling. Methods take protobuf `RpcController`/`Closure` and `brpc::flatbuffers::Message` parameters. Derive from `Echo` and override the schema methods; supply an implementation of -`brpc::flatbuffers::RpcChannel` to a stub. Channel ownership defaults to borrowed; -`Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. +`brpc::flatbuffers::RpcChannel` to a stub, such as a `brpc::Channel` initialized +with `ChannelOptions::protocol = "fb_rpc"`. Register the implementation with +`Server::AddFlatBuffersService`. See [FlatBuffers RPC](../../docs/en/flatbuffers.md) +for ownership, protocol limitations, and response verification. Channel ownership +defaults to borrowed; `Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. This intentionally bounded generator supports unary table RPCs, multiple services/methods, namespaces, included request/response tables, and absent @@ -93,6 +107,10 @@ when possible; if rollback also fails, the generator attempts to remove both final outputs and reports that manual cleanup is needed when the filesystem refuses. The standalone generator currently requires POSIX file-locking semantics, including on Linux and macOS. +Service header guards preserve the exact filename and first exported service's +fully-qualified identity. They distinguish punctuation/case and same basenames +in different namespaces without depending on checkout or output paths. Schemas +included together must still define distinct fully-qualified C++ services. ## Descriptor and completion contracts @@ -202,12 +220,13 @@ Ordinary FlatBuffers-OFF jobs are not codegen evidence. No root build file needs modification. The default standalone build enables CTest and additionally needs matching official `flatc`, protobuf, and gflags -development headers: +development headers. The commands below use CMake/CTest 3.17+ for +`--no-tests=error` (the standalone generator itself has a CMake 3.10 minimum): ```sh cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=ON cmake --build /tmp/brpc-codegen-build -j2 -ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +(cd /tmp/brpc-codegen-build && ctest --no-tests=error --output-on-failure) ``` The build compiles the generated service against the real bRPC headers. CTest @@ -223,19 +242,34 @@ source tree. The name-shadowing cases are compile-only by default; pass `RUNTIME_LIBRARIES` when invoking `acceptance.cmake` directly to exercise their runtime checks as well. +Expect **one** CTest named `flatbuffers_codegen_acceptance` in this mode. Its +inner checks also compile colliding-basename headers alone and together in both +orders, and compare regenerated bytes after relocating the schema. An empty +CTest selection is not acceptance. + To enable additional behavior tests against an **already built** -FlatBuffers-enabled bRPC library: +FlatBuffers-enabled bRPC library, use matching runtime headers/dependencies and +an existing library. A shared library avoids having to reconstruct all static +platform link dependencies. Build it with root `BUILD_SHARED_LIBS=ON` and target +`brpc-shared` first; use `.dylib` on macOS instead of `.so`: ```sh +BRPC_LIBRARY=/absolute/path/to/FB-on/output/lib/libbrpc.so cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build \ -DBUILD_TESTING=ON \ - -DBRPC_CODEGEN_BRPC_LIBRARY=/path/to/libbrpc.a + -DBRPC_CODEGEN_BRPC_LIBRARY="$BRPC_LIBRARY" cmake --build /tmp/brpc-codegen-build -j2 -ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +(cd /tmp/brpc-codegen-build && ctest --no-tests=error --output-on-failure) ``` -Use `BRPC_CODEGEN_EXTRA_LIBRARIES` for additional dependencies of a custom bRPC -build. The runtime test covers concurrent first descriptor access, sparse +Expect **two** CTests in this mode: `flatbuffers_codegen_acceptance` and +`flatbuffers_codegen_runtime`. For non-default dependency installations, pass +their prefixes through `CMAKE_PREFIX_PATH`; modern Protobuf needs its CMake +config package and Abseil. A static `libbrpc.a` is also accepted, but the caller +must supply its complete transitive/platform link requirements (including macOS +frameworks where needed) via `BRPC_CODEGEN_EXTRA_LIBRARIES`. + +The runtime test covers concurrent first descriptor access, sparse stub/dispatch mapping, absent and present strings, invalid/foreign/forged methods, failed request verification, null arguments, unimplemented methods, exactly-once error callbacks (including self-deleting closures), deferred