Merge branch 'google3'

Change-Id: Ib831e8d42afbaf150c39efea539eadc64b9ab378

script/oss.py

@@ -1,9 +1,61 @@
 #!/usr/bin/python3
 
+import argparse
 import os
 import shutil
 import sys
 
+copy_header="""Copyright 2020 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.""".split("\n")
+
+MISSING = 0
+HEADLINE = 1
+PARTIAL = 2
+FULL = 3
+
+def has_copyright(lines, max_lookup=3):
+  pos = 0
+  for line in lines:
+    pos += 1 # points to the next line
+    if line.find(copy_header[0]) >= 0:
+      break
+    if pos > max_lookup:
+      return MISSING
+
+  result = HEADLINE
+
+  for line in copy_header[1:]:
+    if lines[pos].find(line) < 0:
+      return result
+    else:
+      result = PARTIAL
+
+  return FULL
+
+def add_copyright(lines, prefix, offset):
+  new_lines = lines[0:offset]
+  if offset:
+    new_lines.append("\n")
+  for line in copy_header:
+    if line:
+      new_lines.append(prefix + " " + line + "\n")
+    else:
+      new_lines.append(prefix + "\n")
+  new_lines.append("\n")
+  new_lines.extend(lines[offset:])
+  return new_lines
+
 def copy_files_to_oss_project(src_root, dst_root):
   shutil.rmtree(dst_root + "/cpp", ignore_errors=True)
   shutil.rmtree(dst_root + "/proto", ignore_errors=True)
@@ -12,10 +64,27 @@ def copy_files_to_oss_project(src_root, dst_root):
   shutil.copytree(src_root + "/connections/core/", dst_root + "/cpp/core/")
   shutil.copytree(src_root + "/connections/proto/", dst_root + "/proto/connections/")
 
-def post_process_oss_files(path):
+def detect_file_copy_header_options(fname, lines):
+  if not lines:
+    return None # ignore empty file
+  suffixes = [".cc", ".cpp", ".cxx", ".c", ".h", ".hpp", ".inc", ".proto"]
+  for suffix in suffixes:
+    if fname.endswith(suffix):
+      return ("//", 0)
+  if (fname in ["CMakeLists.txt", "BUILD.gn", "BUILD"]) or (
+              fname.startswith("CMakeLists") or fname.endswith(".cmake")):
+    return ("#", 0)
+  if lines[0].startswith("#!"):
+    return ("#", 1)
+  return None
+
+def post_process_oss_files(path, args):
   modified_total = 0
   top_level = True
-  top_dirs = ["cpp", "proto"]
+  if args.all:
+    top_level = False # no special actions to take at top level
+  else:
+    top_dirs = ["cpp", "proto"]
   transforms = (
     ("third_party/", ""),
     ("location/nearby/connections/core", "core"),
@@ -23,7 +92,8 @@ def post_process_oss_files(path):
     ("security/cryptauth/lib/securegcm", "securegcm"),
     ("testing/base/public/gmock.h", "gmock/gmock.h"),
     ("testing/base/public/gunit.h", "gtest/gtest.h"),
-    ("net/proto2/compat/public/message_lite.h", "google/protobuf/message_lite.h"),
+    ("net/proto2/compat/public/message_lite.h",
+     "google/protobuf/message_lite.h"),
     ("LOCATION_NEARBY_CONNECTIONS_", ""),
     ("LOCATION_NEARBY_CPP_", ""),
     ("location/nearby/proto", "proto"),
@@ -32,7 +102,7 @@ def post_process_oss_files(path):
     (".proto.h", ".pb.h"),
   )
   for root, dirs, files in os.walk(path):
-    if top_level:
+    if top_level and top_dirs:
       # we must convert cpp/ and proto/ subtrees.
       # everything else is not parsed.
       dirs.clear()
@@ -41,32 +111,84 @@ def post_process_oss_files(path):
       continue
     for file in files:
       fname = root + "/" + file
+      print("parsing: {}".format(fname))
       if file in ["METADATA"]:
         os.remove(fname)
         continue
       modified = False
       lines=[]
+      google3_ignore = False
       with open(fname, "r") as f:
         for line in f:
           orig = line
-          for lookup, substitute in transforms:
+
-            line = line.replace(lookup, substitute)
+          if not args.no_subst:
-          if orig != line:
+            for lookup, substitute in transforms:
-            modified = True
+              line = line.replace(lookup, substitute)
+            if orig != line:
+              modified = True
+
+          if args.google3_filter:
+            if line.find("nearby:google3-only") >= 0:
+              modified = True
+              continue
+            if line.find("nearby:google3-begin") >= 0:
+              modified = True
+              google3_ignore = True
+              continue
+            if line.find("nearby:google3-end") >= 0:
+              modified = True
+              google3_ignore = False
+              continue
+            if google3_ignore:
+              modified = True
+              continue
           lines.append(line)
+
+      if args.fix_oss_headers:
+        options = detect_file_copy_header_options(file, lines)
+        if options is not None:
+          if not has_copyright(lines):
+            prefix, offset = options
+            lines = add_copyright(lines, prefix, offset)
+            modified = True
       if modified:
         with open(fname, "w") as f:
           for line in lines:
             f.write(line)
         modified_total += 1
+    if args.no_recurse:
+      break
+
   return modified_total
 
-def main(args):
+def main():
-  src = "/google/src/cloud/%s/%s/google3/location/nearby" % (os.environ["USER"], args[1])
+  parser = argparse.ArgumentParser('Opensource Nearby Release Tool')
-  dst = args[2]
+  parser.add_argument('target', action='store', type=str, nargs="+", default=[])
-  copy_files_to_oss_project(src, dst)
+  parser.add_argument('--workspace', action='store', default="")
-  total = post_process_oss_files(dst)
+  parser.add_argument('--all', action='store_true', default=False)
+  parser.add_argument('--fix-oss-headers', action='store_true', default=False)
+  parser.add_argument('--google3-filter', action='store_true', default=False)
+  parser.add_argument('--no-copy', action='store_true', default=False)
+  parser.add_argument('--no-subst', action='store_true', default=False)
+  parser.add_argument('--no-recurse', action='store_true', default=False)
+  args = parser.parse_args()
+  if args.google3_filter:
+    print("google3-specific code will be removed")
+  if args.workspace:
+    src = "/google/src/cloud/%s/%s/google3/location/nearby" % (os.environ["USER"], args.workspace)
+  else:
+     args.no_copy = True
+  if len(args.target) == 1:
+    dst = args.target[0]
+  else:
+    args.no_copy = True
+    args.all = True
+  if not args.no_copy: copy_files_to_oss_project(src, dst)
+  total = 0
+  for dst in args.target:
+    total += post_process_oss_files(dst, args)
   print("Total modified: {} files".format(total))
 
 if __name__ == "__main__":
-  sys.exit(main(sys.argv))
+  sys.exit(main())