Merge #15257: Scripts and tools: Bump flake8 to 3.7.8

3d0a82cff8 devtools: Accomodate block-style copyright blocks (Ben Woosley)
0ef0e51fe4 lint: Bump flake8 to 3.7.8 (Ben Woosley)
838920704a lint: Disable flake8 W504 warning (Ben Woosley)
b21680baf5 test/contrib: Fix invalid escapes in regex strings (Ben Woosley)

Pull request description:

  This is a second go at #15221, fixing new lints in:
  W504 line break after binary operator
  W605 invalid escape sequence
  F841 local variable 'e' is assigned to but never used

  This time around:
  * One commit per rule, for easier review
  * I went with the PEP-8 style of breaking before binary operators
  * I looked into the raw regex newline issue, and found that raw strings with newlines embedded do work appropriately. E.g. run `re.match(r" \n ", " \n ")` to check this for yourself. `re.MULTILINE` exists to modify `^` and `$` in multiline scenarios, but  all of these searches are per-line.

ACKs for top commit:
  practicalswift:
    ACK 3d0a82cff8 -- diff looks correct

Tree-SHA512: bea0c144cadd72e4adf2e9a4b4ee0535dd91a8e694206924cf8a389dc9253f364a717edfe9abda88108fbb67fda19b9e823f46822d7303c0aaa72e48909a6105

ci/lint/04_install.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C
 
 travis_retry pip3 install codespell==1.15.0
-travis_retry pip3 install flake8==3.5.0
+travis_retry pip3 install flake8==3.7.8
 travis_retry pip3 install vulture==0.29
 
 SHELLCHECK_VERSION=v0.6.0

contrib/devtools/clang-format-diff.py

@@ -106,7 +106,7 @@ def main():
   filename = None
   lines_by_file = {}
   for line in sys.stdin:
-    match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
+    match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
     if match:
       filename = match.group(2)
     if filename is None:
@@ -119,7 +119,7 @@ def main():
       if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
         continue
 
-    match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
+    match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line)
     if match:
       start_line = int(match.group(1))
       line_count = 1

contrib/devtools/copyright_header.py

@@ -71,7 +71,7 @@ def get_filenames_to_examine(base_directory):
 ################################################################################
 
 
-COPYRIGHT_WITH_C = 'Copyright \(c\)'
+COPYRIGHT_WITH_C = r'Copyright \(c\)'
 COPYRIGHT_WITHOUT_C = 'Copyright'
 ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)
 
@@ -85,21 +85,21 @@ ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE,
 ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE)
 
 def compile_copyright_regex(copyright_style, year_style, name):
-    return re.compile('%s %s,? %s' % (copyright_style, year_style, name))
+    return re.compile(r'%s %s,? %s( +\*)?\n' % (copyright_style, year_style, name))
 
 EXPECTED_HOLDER_NAMES = [
-    "Satoshi Nakamoto\n",
+    r"Satoshi Nakamoto",
-    "The Bitcoin Core developers\n",
+    r"The Bitcoin Core developers",
-    "BitPay Inc\.\n",
+    r"BitPay Inc\.",
-    "University of Illinois at Urbana-Champaign\.\n",
+    r"University of Illinois at Urbana-Champaign\.",
-    "Pieter Wuille\n",
+    r"Pieter Wuille",
-    "Wladimir J. van der Laan\n",
+    r"Wladimir J\. van der Laan",
-    "Jeff Garzik\n",
+    r"Jeff Garzik",
-    "Jan-Klaas Kollhof\n",
+    r"Jan-Klaas Kollhof",
-    "ArtForz -- public domain half-a-node\n",
+    r"ArtForz -- public domain half-a-node",
-    "Intel Corporation",
+    r"Intel Corporation ?",
-    "The Zcash developers",
+    r"The Zcash developers",
-    "Jeremy Rubin",
+    r"Jeremy Rubin",
 ]
 
 DOMINANT_STYLE_COMPILED = {}
@@ -329,7 +329,7 @@ def write_file_lines(filename, file_lines):
 # update header years execution
 ################################################################################
 
-COPYRIGHT = 'Copyright \(c\)'
+COPYRIGHT = r'Copyright \(c\)'
 YEAR = "20[0-9][0-9]"
 YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR)
 HOLDER = 'The Bitcoin Core developers'

contrib/devtools/symbol-check.py

@@ -141,7 +141,7 @@ def read_libraries(filename):
     for line in stdout.splitlines():
         tokens = line.split()
         if len(tokens)>2 and tokens[1] == '(NEEDED)':
-            match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
+            match = re.match(r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
             if match:
                 libraries.append(match.group(1))
             else:
@@ -171,5 +171,3 @@ if __name__ == '__main__':
                 retval = 1
 
     sys.exit(retval)
-
-

contrib/linearize/linearize-data.py

@@ -263,12 +263,12 @@ if __name__ == '__main__':
     f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
-        m = re.search('^\s*#', line)
+        m = re.search(r'^\s*#', line)
         if m:
             continue
 
         # parse key=value lines
-        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
+        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
         if m is None:
             continue
         settings[m.group(1)] = m.group(2)

contrib/linearize/linearize-hashes.py

@@ -106,12 +106,12 @@ if __name__ == '__main__':
     f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
-        m = re.search('^\s*#', line)
+        m = re.search(r'^\s*#', line)
         if m:
             continue
 
         # parse key=value lines
-        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
+        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
         if m is None:
             continue
         settings[m.group(1)] = m.group(2)

contrib/macdeploy/macdeployqtplus

@@ -766,7 +766,7 @@ if config.dmg is not None:
         except subprocess.CalledProcessError as e:
             sys.exit(e.returncode)
 
-        m = re.search("/Volumes/(.+$)", output)
+        m = re.search(r"/Volumes/(.+$)", output)
         disk_root = m.group(0)
         disk_name = m.group(1)
         

contrib/seeds/generate-seeds.py

@@ -74,7 +74,7 @@ def name_to_ipv6(addr):
         raise ValueError('Could not parse address %s' % addr)
 
 def parse_spec(s, defaultport):
-    match = re.match('\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
+    match = re.match(r'\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
     if match: # ipv6
         host = match.group(1)
         port = match.group(2)
@@ -136,4 +136,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-

test/functional/combine_logs.py

@@ -81,7 +81,7 @@ def read_logs(tmp_dir):
     chain = glob.glob("{}/node0/*/debug.log".format(tmp_dir))
     if chain:
         chain = chain[0]  # pick the first one if more than one chain was found (should never happen)
-        chain = re.search('node0/(.+?)/debug\.log$', chain).group(1)  # extract the chain name
+        chain = re.search(r'node0/(.+?)/debug\.log$', chain).group(1)  # extract the chain name
     else:
         chain = 'regtest'  # fallback to regtest (should only happen when none exists)
 

test/functional/feature_logging.py

@@ -36,7 +36,7 @@ class LoggingTest(BitcoinTestFramework):
         invdir = self.relative_log_path("foo")
         invalidname = os.path.join("foo", "foo.log")
         self.stop_node(0)
-        exp_stderr = "Error: Could not open debug log file \S+$"
+        exp_stderr = r"Error: Could not open debug log file \S+$"
         self.nodes[0].assert_start_raises_init_error(["-debuglogfile=%s" % (invalidname)], exp_stderr, match=ErrorMatch.FULL_REGEX)
         assert not os.path.isfile(os.path.join(invdir, "foo.log"))
 

test/functional/feature_uacomment.py

@@ -27,12 +27,12 @@ class UacommentTest(BitcoinTestFramework):
 
         self.log.info("test -uacomment max length")
         self.stop_node(0)
-        expected = "Error: Total length of network version string \([0-9]+\) exceeds maximum length \(256\). Reduce the number or size of uacomments."
+        expected = r"Error: Total length of network version string \([0-9]+\) exceeds maximum length \(256\). Reduce the number or size of uacomments."
         self.nodes[0].assert_start_raises_init_error(["-uacomment=" + 'a' * 256], expected, match=ErrorMatch.FULL_REGEX)
 
         self.log.info("test -uacomment unsafe characters")
         for unsafe_char in ['/', ':', '(', ')', '₿', '🏃']:
-            expected = "Error: User Agent comment \(" + re.escape(unsafe_char) + "\) contains unsafe characters."
+            expected = r"Error: User Agent comment \(" + re.escape(unsafe_char) + r"\) contains unsafe characters."
             self.nodes[0].assert_start_raises_init_error(["-uacomment=" + unsafe_char], expected, match=ErrorMatch.FULL_REGEX)
 
 

test/functional/wallet_multiwallet.py

@@ -93,12 +93,12 @@ class MultiWalletTest(BitcoinTestFramework):
 
         # should not initialize if one wallet is a copy of another
         shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy'))
-        exp_stderr = "BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
+        exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
         self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         # should not initialize if wallet file is a symlink
         os.symlink('w8', wallet_dir('w8_symlink'))
-        self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
+        self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
 
         # should not initialize if the specified walletdir does not exist
         self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
@@ -139,7 +139,7 @@ class MultiWalletTest(BitcoinTestFramework):
         competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir')
         os.mkdir(competing_wallet_dir)
         self.restart_node(0, ['-walletdir=' + competing_wallet_dir])
-        exp_stderr = "Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
+        exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
         self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         self.restart_node(0, extra_args)

test/lint/check-doc.py

@@ -15,8 +15,8 @@ import re
 
 FOLDER_GREP = 'src'
 FOLDER_TEST = 'src/test/'
-REGEX_ARG = '(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
+REGEX_ARG = r'(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
-REGEX_DOC = 'AddArg\("(-[^"=]+?)(?:=|")'
+REGEX_DOC = r'AddArg\("(-[^"=]+?)(?:=|")'
 CMD_ROOT_DIR = '$(git rev-parse --show-toplevel)/{}'.format(FOLDER_GREP)
 CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST)
 CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWalletOptions' -- {} | grep AddArg".format(CMD_ROOT_DIR)

test/lint/check-rpc-mappings.py

@@ -48,13 +48,13 @@ def process_commands(fname):
         for line in f:
             line = line.rstrip()
             if not in_rpcs:
-                if re.match("static const CRPCCommand .*\[\] =", line):
+                if re.match(r"static const CRPCCommand .*\[\] =", line):
                     in_rpcs = True
             else:
                 if line.startswith('};'):
                     in_rpcs = False
                 elif '{' in line and '"' in line:
-                    m = re.search('{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
+                    m = re.search(r'{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
                     assert m, 'No match to table expression: %s' % line
                     name = parse_string(m.group(2))
                     args_str = m.group(4).strip()
@@ -80,7 +80,7 @@ def process_mapping(fname):
                 if line.startswith('};'):
                     in_rpcs = False
                 elif '{' in line and '"' in line:
-                    m = re.search('{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
+                    m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
                     assert m, 'No match to table expression: %s' % line
                     name = parse_string(m.group(1))
                     idx = int(m.group(2))

test/lint/lint-format-strings.py

@@ -55,7 +55,7 @@ def normalize(s):
     assert type(s) is str
     s = s.replace("\n", " ")
     s = s.replace("\t", " ")
-    s = re.sub("/\*.*?\*/", " ", s)
+    s = re.sub(r"/\*.*?\*/", " ", s)
     s = re.sub(" {2,}", " ", s)
     return s.strip()
 

test/lint/lint-python.sh

@@ -73,7 +73,6 @@ enabled=(
     W291 # trailing whitespace
     W292 # no newline at end of file
     W293 # blank line contains whitespace
-    W504 # line break after binary operator
     W601 # .has_key() is deprecated, use "in"
     W602 # deprecated form of raising exception
     W603 # "<>" is deprecated, use "!="