Refactoring script: Copy sys_util(_core) to base

This will allow us to merge sys_util and win_sys_util.
The original source locations will remain until they
are moved into the ChromeOS source base.

See go/future-of-sys-util for details

BUG=b:22320646
TEST=run it

Change-Id: Ie1adcadba9d35f18cc7472dc78f05262114a726f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3536890
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>

base/Cargo.toml

@@ -18,3 +18,4 @@ smallvec = "1.6.1"
 sync = { path = "../common/sync" }
 sys_util = { path = "../common/sys_util" }
 thiserror = "1.0.20"
+tempfile = "3"

io_uring/DEPRECATED.md

@@ -0,0 +1,4 @@
+Use crosvm/io_uring instead.
+
+Code in this directory is not used by crosvm, it is only used in ChromeOS and will move to a
+separate ChromeOS repository soon.

tools/contrib/cargo_refactor.py

@@ -8,32 +8,38 @@
 # Contains the last run refactoring for reference. Don't run this script, it'll
 # fail, but use it as a foundation for other refactorings.
 
+from contextlib import contextmanager
 from pathlib import Path
 import os
 import re
 import shutil
 import subprocess
-from typing import List, Tuple, Union
+from typing import Callable, List, Tuple, Union
+
 
 SearchPattern = Union[str, re.Pattern[str]]
+Replacement = Union[str, Callable[[re.Match[str]], str]]
 
 
-def replace_in_file(file_path: Path, search: SearchPattern, replace: str):
+def append_to_file(file_path: Path, appendix: str):
+    contents = file_path.read_text()
+    file_path.write_text(contents.rstrip() + "\n" + appendix + "\n")
+
+
+def replace_in_file(file_path: Path, search: SearchPattern, replace: Replacement):
     if not file_path.exists():
         print(f"WARNING: Does not exist {file_path}")
         return
     if isinstance(search, str):
         search = re.escape(search)
-    with open(file_path, "r") as file:
-        contents = file.read()
+    contents = file_path.read_text()
     (contents, count) = re.subn(search, replace, contents)
     if count > 0:
         print(f"replacing '{search}' with '{replace}' in {file_path}")
-        with open(file_path, "w") as file:
-            file.write(contents)
+        file_path.write_text(contents)
 
 
-def replace_in_files(glob: str, replacements: List[Tuple[SearchPattern, str]]):
+def replace_in_files(glob: str, replacements: List[Tuple[SearchPattern, Replacement]]):
     for file in Path().glob(glob):
         for (search, replace) in replacements:
             replace_in_file(file, search, replace)
@@ -50,13 +56,11 @@ def replace_path_in_all_cargo_toml(old_path: Path, new_path: Path):
 
 def update_path_deps(toml: Path, from_path: Path, to_path: Path):
     "Update path deps in toml file after moving it"
-    with open(toml, "r") as file:
-        contents = file.read()
+    contents = toml.read_text()
     for old_dep in re.findall('{ path = "([^"]+)"', contents):
         new_dep = os.path.relpath((from_path / old_dep).resolve(), to_path)
         contents = contents.replace(f'path = "{old_dep}"', f'path = "{new_dep}"')
-    with open(toml, "w") as file:
-        file.write(contents)
+    toml.write_text(contents)
 
 
 def move_crate(from_path: Path, to_path: Path):
@@ -88,14 +92,119 @@ def update_workspace_members():
     replace_in_file(Path("Cargo.toml"), re.compile(r"exclude = \[[^\]]+\]"), "\n".join(exclude))
 
 
+@contextmanager
+def chdir(path: Union[Path, str]):
+    origin = Path().absolute()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(origin)
+
+
+def copy_crate_src_to_module(source: str, destination: str):
+    shutil.rmtree(destination, ignore_errors=True)
+    shutil.copytree(source, destination)
+    with chdir(destination):
+        Path("lib.rs").rename("mod.rs")
+
+
 def main():
     os.chdir(Path(__file__).parent.parent.parent)
 
-    move_crate(Path("common/io_uring"), Path("io_uring"))
-    replace_in_file(Path("io_uring/Cargo.toml"), "[workspace]", "")
-    replace_in_file(Path("io_uring/bindgen.sh"), "common/io_uring", "io_uring")
+    subprocess.check_call(["git", "checkout", "-f", "--", "base"])
 
-    update_workspace_members()
+    # Move crates to base
+    copy_crate_src_to_module("common/sys_util_core/src", "base/src/common")
+    copy_crate_src_to_module("common/sys_util/src", "base/src/unix")
+
+    # Move poll_token_derive. Rename it so it won't conflict with the version used by ChromeOS.
+    move_crate(Path("common/sys_util_core/poll_token_derive"), Path("base/base_poll_token_derive"))
+    append_to_file(
+        Path("base/Cargo.toml"),
+        'base_poll_token_derive = { path = "base_poll_token_derive" }',
+    )
+    replace_in_file(
+        Path("base/base_poll_token_derive/Cargo.toml"),
+        'name = "poll_token_derive"',
+        'name = "base_poll_token_derive"',
+    )
+
+    # Import the new modules
+    replace_in_file(
+        Path("base/src/lib.rs"),
+        "mod event;",
+        "pub mod unix;\npub mod common;\nmod event;",
+    )
+
+    # Flatten all imports for easier replacements
+    subprocess.check_call(
+        ["rustfmt", "+nightly", "--config=imports_granularity=item", "base/src/lib.rs"]
+    )
+
+    # Fixup macros since they like to have special treatement.
+    macros = [
+        "debug",
+        "error",
+        "handle_eintr_errno",
+        "info",
+        "ioctl_io_nr",
+        "ioctl_ior_nr",
+        "ioctl_iow_nr",
+        "ioctl_iowr_nr",
+        "syscall",
+        "warn",
+        "volatile_at_impl",
+        "volatile_impl",
+        "generate_scoped_event",
+    ]
+    for macro in macros:
+        # Update use statments. #[macro_export] exports them on the crate scoped
+        replace_in_files(
+            "base/src/**/*.rs",
+            [
+                (f"sys_util::{macro}", f"crate::{macro}"),
+                (f"sys_util_core::{macro}", f"crate::{macro}"),
+                (f"super::{macro}", f"crate::{macro}"),
+            ],
+        )
+        # We do not need to import them in lib.rs, they are in the same scope already.
+        replace_in_files(
+            "base/src/lib.rs",
+            [
+                (f"pub use crate::{macro};\n", ""),
+                (f"use crate::{macro};\n", ""),
+            ],
+        )
+
+    # Replace $crate:: with $crate::unix/common (unless it's a macro invocation..)
+    for sys in ("unix", "common"):
+
+        def replace_references_in_macros(match: re.Match[str]):
+            name = match.group(0)
+            if not name.endswith("!"):
+                return name.replace("$crate", f"$crate::{sys}")
+            return name
+
+        replace_in_files(
+            f"base/src/{sys}/**/*.rs",
+            [(re.compile(r"([\w\*\_\$]+\:\:)+([\w\*\_\!]+)"), replace_references_in_macros)],
+        )
+
+    # Update references to the above crates in base:
+    replace_in_files(
+        "base/src/**/*.rs",
+        [
+            ("sys_util_core::", "crate::common::"),
+            ("sys_util::", "crate::unix::"),
+            ("poll_token_derive::", "base_poll_token_derive::"),
+        ],
+    )
+
+    # Unflatten imports again
+    subprocess.check_call(
+        ["rustfmt", "+nightly", "--config=imports_granularity=crate", "base/src/lib.rs"]
+    )
 
 
 main()

tools/impl/test_config.py

@@ -39,6 +39,7 @@ class TestOption(enum.Enum):
 # Please add a bug number when restricting a tests.
 if os.name == "posix":
     CRATE_OPTIONS: dict[str, list[TestOption]] = {
+        "base": [TestOption.SINGLE_THREADED, TestOption.LARGE],
         "cros_async": [TestOption.LARGE],
         "crosvm_plugin": [TestOption.DO_NOT_BUILD_AARCH64, TestOption.DO_NOT_BUILD_ARMHF],
         "crosvm": [TestOption.SINGLE_THREADED],