Commit 82d6df58 authored by Yedidya Feldblum's avatar Yedidya Feldblum Committed by Facebook Github Bot

Generate escape tables at compile time in C++

Summary: [Folly] Generate format tables at compile time in C++, rather than as a separate custom build step in Python.

Reviewed By: ot

Differential Revision: D6830372

fbshipit-source-id: 25770676e59c8070eaef7cbb691a5ba1c4b0a8f8
parent 7f7abd9f
......@@ -30,6 +30,5 @@ folly/m4/ltsugar.m4
folly/m4/ltversion.m4
folly/m4/lt~obsolete.m4
folly/generate_fingerprint_tables
folly/EscapeTables.cpp
folly/GroupVarintTables.cpp
folly/FingerprintTables.cpp
......@@ -57,18 +57,6 @@ endif()
set(FOLLY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/folly")
# Generate a few tables and create the main config file.
find_package(PythonInterp REQUIRED)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/folly/build/EscapeTables.cpp
COMMAND
${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/folly/build
COMMAND
${PYTHON_EXECUTABLE} "${FOLLY_DIR}/build/generate_escape_tables.py"
--install_dir ${CMAKE_CURRENT_BINARY_DIR}/folly/build
DEPENDS ${FOLLY_DIR}/build/generate_escape_tables.py
COMMENT "Generating the escape tables..." VERBATIM
)
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/folly/build/GroupVarintTables.cpp"
COMMAND
......@@ -201,7 +189,6 @@ endif()
add_library(folly_base OBJECT
${files} ${hfiles}
${CMAKE_CURRENT_BINARY_DIR}/folly/folly-config.h
${CMAKE_CURRENT_BINARY_DIR}/folly/build/EscapeTables.cpp
${CMAKE_CURRENT_BINARY_DIR}/folly/build/GroupVarintTables.cpp
)
if (BUILD_SHARED_LIBS)
......@@ -213,7 +200,6 @@ target_include_directories(folly_base PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
# Add the generated files to the correct source group.
source_group("folly" FILES ${CMAKE_CURRENT_BINARY_DIR}/folly/folly-config.h)
source_group("folly\\build" FILES
${CMAKE_CURRENT_BINARY_DIR}/folly/build/EscapeTables.cpp
${CMAKE_CURRENT_BINARY_DIR}/folly/build/FingerprintTables.cpp
${CMAKE_CURRENT_BINARY_DIR}/folly/build/GroupVarintTables.cpp
)
......
......@@ -483,10 +483,6 @@ nobase_follyinclude_HEADERS = \
Utility.h \
Varint.h
EscapeTables.cpp: build/generate_escape_tables.py
$(PYTHON) build/generate_escape_tables.py
CLEANFILES += EscapeTables.cpp
GroupVarintTables.cpp: build/generate_varint_tables.py
$(PYTHON) build/generate_varint_tables.py
CLEANFILES += GroupVarintTables.cpp
......@@ -500,7 +496,6 @@ libfollybase_la_SOURCES = \
Conv.cpp \
Demangle.cpp \
detail/RangeCommon.cpp \
EscapeTables.cpp \
Format.cpp \
FormatArg.cpp \
memory/MallctlHelper.cpp \
......
......@@ -32,7 +32,7 @@ namespace detail {
// ('\n' = 10 maps to 'n'), 'O' if the character should be printed as
// an octal escape sequence, or 'P' if the character is printable and
// should be printed as is.
extern const char cEscapeTable[];
extern const std::array<char, 256> cEscapeTable;
} // namespace detail
template <class String>
......@@ -76,10 +76,10 @@ namespace detail {
// ('n' maps to 10 = '\n'), 'O' if this is the first character of an
// octal escape sequence, 'X' if this is the first character of a
// hexadecimal escape sequence, or 'I' if this escape sequence is invalid.
extern const char cUnescapeTable[];
extern const std::array<char, 256> cUnescapeTable;
// Map from the character code to the hex value, or 16 if invalid hex char.
extern const unsigned char hexTable[];
extern const std::array<unsigned char, 256> hexTable;
} // namespace detail
template <class String>
......@@ -157,7 +157,7 @@ namespace detail {
// 2 = pass through in PATH mode
// 3 = space, replace with '+' in QUERY mode
// 4 = percent-encode
extern const unsigned char uriEscapeTable[];
extern const std::array<unsigned char, 256> uriEscapeTable;
} // namespace detail
template <class String>
......
......@@ -26,6 +26,7 @@
#include <glog/logging.h>
#include <folly/ScopeGuard.h>
#include <folly/container/Array.h>
namespace folly {
......@@ -35,6 +36,92 @@ static_assert(IsConvertible<bool>::value, "");
static_assert(IsConvertible<int>::value, "");
static_assert(!IsConvertible<std::vector<int>>::value, "");
namespace detail {
struct string_table_c_escape_make_item {
constexpr char operator()(std::size_t index) const {
// clang-format off
return
index == '"' ? '"' :
index == '\\' ? '\\' :
index == '?' ? '?' :
index == '\n' ? 'n' :
index == '\r' ? 'r' :
index == '\t' ? 't' :
index < 32 || index > 126 ? 'O' : // octal
'P'; // printable
// clang-format on
}
};
struct string_table_c_unescape_make_item {
constexpr char operator()(std::size_t index) const {
// clang-format off
return
index == '\'' ? '\'' :
index == '?' ? '?' :
index == '\\' ? '\\' :
index == '"' ? '"' :
index == 'a' ? '\a' :
index == 'b' ? '\b' :
index == 'f' ? '\f' :
index == 'n' ? '\n' :
index == 'r' ? '\r' :
index == 't' ? '\t' :
index == 'v' ? '\v' :
index >= '0' && index <= '7' ? 'O' : // octal
index == 'x' ? 'X' : // hex
'I'; // invalid
// clang-format on
}
};
struct string_table_hex_make_item {
constexpr unsigned char operator()(std::size_t index) const {
// clang-format off
return
index >= '0' && index <= '9' ? index - '0' :
index >= 'a' && index <= 'f' ? index - 'a' + 10 :
index >= 'A' && index <= 'F' ? index - 'A' + 10 :
16;
// clang-format on
}
};
struct string_table_uri_escape_make_item {
// 0 = passthrough
// 1 = unused
// 2 = safe in path (/)
// 3 = space (replace with '+' in query)
// 4 = always percent-encode
constexpr unsigned char operator()(std::size_t index) const {
// clang-format off
return
index >= '0' && index <= '9' ? 0 :
index >= 'A' && index <= 'Z' ? 0 :
index >= 'a' && index <= 'z' ? 0 :
index == '-' ? 0 :
index == '_' ? 0 :
index == '.' ? 0 :
index == '~' ? 0 :
index == '/' ? 2 :
index == ' ' ? 3 :
4;
// clang-format on
}
};
constexpr decltype(cEscapeTable) cEscapeTable =
make_array_with<256>(string_table_c_escape_make_item{});
constexpr decltype(cUnescapeTable) cUnescapeTable =
make_array_with<256>(string_table_c_unescape_make_item{});
constexpr decltype(hexTable) hexTable =
make_array_with<256>(string_table_hex_make_item{});
constexpr decltype(uriEscapeTable) uriEscapeTable =
make_array_with<256>(string_table_uri_escape_make_item{});
} // namespace detail
static inline bool is_oddspace(char c) {
return c == '\n' || c == '\t' || c == '\r';
}
......
#!/usr/bin/env python
#
# Generate Escape tables.
# Copyright 2011 Facebook
#
# @author Tudor Bosman (tudorb@fb.com)
#
import os
from optparse import OptionParser
OUTPUT_FILE = "EscapeTables.cpp"
def generate(f):
f.write("namespace folly {\n"
"namespace detail {\n"
"\n")
f.write("extern const char cEscapeTable[] =\n")
escapes = dict((
('"', '\\"'),
('\\', '\\\\'),
('?', '?'),
('\n', 'n'),
('\r', 'r'),
('\t', 't'),
))
for i in range(0, 256):
if i % 64 == 0:
if i != 0:
f.write("\"\n")
f.write(" \"")
c = chr(i)
if c in escapes:
c = escapes[c]
elif i < 32 or i > 126:
c = 'O' # octal
else:
c = 'P' # printable
f.write(c)
f.write("\";\n\n")
f.write("extern const char cUnescapeTable[] =\n")
for i in range(0, 256):
if i % 64 == 0:
if i != 0:
f.write("\"\n")
f.write(" \"")
c = chr(i)
if c in '\'?':
f.write(c)
elif c in '"\\abfnrtv':
f.write("\\" + c)
elif i >= ord('0') and i <= ord('7'):
f.write("O") # octal
elif c == "x":
f.write("X") # hex
else:
f.write("I") # invalid
f.write("\";\n\n")
f.write("extern const unsigned char hexTable[] = {")
for i in range(0, 256):
if i % 16 == 0:
f.write("\n ")
if i >= ord('0') and i <= ord('9'):
f.write("{0:2d}, ".format(i - ord('0')))
elif i >= ord('a') and i <= ord('f'):
f.write("{0:2d}, ".format(i - ord('a') + 10))
elif i >= ord('A') and i <= ord('F'):
f.write("{0:2d}, ".format(i - ord('A') + 10))
else:
f.write("16, ")
f.write("\n};\n\n")
# 0 = passthrough
# 1 = unused
# 2 = safe in path (/)
# 3 = space (replace with '+' in query)
# 4 = always percent-encode
f.write("extern const unsigned char uriEscapeTable[] = {")
passthrough = (
list(map(ord, '0123456789')) +
list(map(ord, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')) +
list(map(ord, 'abcdefghijklmnopqrstuvwxyz')) +
list(map(ord, '-_.~')))
for i in range(0, 256):
if i % 16 == 0:
f.write("\n ")
if i in passthrough:
f.write("0, ")
elif i == ord('/'):
f.write("2, ")
elif i == ord(' '):
f.write("3, ")
else:
f.write("4, ")
f.write("\n};\n\n")
f.write("} // namespace detail\n"
"} // namespace folly\n")
def main():
parser = OptionParser()
parser.add_option("--install_dir", dest="install_dir", default=".",
help="write output to DIR", metavar="DIR")
parser.add_option("--fbcode_dir")
(options, args) = parser.parse_args()
f = open(os.path.join(options.install_dir, OUTPUT_FILE), "w")
generate(f)
f.close()
if __name__ == "__main__":
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment