Merge pull request #10758 from mkruskal-google/utf8_range
Declare a dependency on utf8_range repo
diff --git a/.gitmodules b/.gitmodules
index 7a46a17..db938cf 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -9,3 +9,6 @@
[submodule "third_party/jsoncpp"]
path = third_party/jsoncpp
url = https://github.com/open-source-parsers/jsoncpp.git
+[submodule "third_party/utf8_range"]
+ path = third_party/utf8_range
+ url = https://github.com/protocolbuffers/utf8_range.git
diff --git a/WORKSPACE b/WORKSPACE
index 82f5230..fb152a2 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -80,6 +80,9 @@
load("@upb//bazel:system_python.bzl", "system_python")
system_python(name = "local_config_python")
+load("@utf8_range//:workspace_deps.bzl", "utf8_range_deps")
+utf8_range_deps()
+
bind(
name = "python_headers",
actual = "@local_config_python//:python_headers",
diff --git a/kokoro/release/ruby/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh
index 12d270e..34a85ab 100755
--- a/kokoro/release/ruby/linux/ruby/ruby_build.sh
+++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh
@@ -10,6 +10,9 @@
cp bazel-bin/protoc .
export PROTOC=$PWD/protoc
+# Pull in dependencies.
+git submodule update --init --recursive
+
umask 0022
pushd ruby
gem install bundler -v 2.1.4
diff --git a/kokoro/release/ruby/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh
index bbfc631..d13c120 100755
--- a/kokoro/release/ruby/macos/ruby/ruby_build.sh
+++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh
@@ -6,6 +6,9 @@
bazel build //:protoc
export PROTOC=$PWD/bazel-bin/protoc
+# Pull in dependencies.
+git submodule update --init --recursive
+
umask 0022
pushd ruby
bundle update && bundle exec rake gem:native
diff --git a/php/BUILD.bazel b/php/BUILD.bazel
index 3c97ad8..ac33702 100644
--- a/php/BUILD.bazel
+++ b/php/BUILD.bazel
@@ -34,7 +34,7 @@
name = "build_extension",
cmd = """
mkdir -p php/ext/google/protobuf/third_party/utf8_range
- cp third_party/utf8_range/* php/ext/google/protobuf/third_party/utf8_range
+ cp external/utf8_range/* php/ext/google/protobuf/third_party/utf8_range
pushd php/ext/google/protobuf
phpize
@@ -48,7 +48,7 @@
name = "extension",
srcs = [
":source_files",
- "//third_party/utf8_range:all_files",
+ "@utf8_range//:utf8_range_srcs",
],
tools = [":build_extension"],
outs = ["protobuf.so"],
diff --git a/php/tests/compile_extension.sh b/php/tests/compile_extension.sh
index dc42aa3..a645ed6 100755
--- a/php/tests/compile_extension.sh
+++ b/php/tests/compile_extension.sh
@@ -4,12 +4,15 @@
cd $(dirname $0)/..
+# Pull in dependencies.
+git submodule update --init --recursive
+
# utf8_range has to live in the base third_party directory.
# We copy it into the ext/google/protobuf directory for the build
# (and for the release to PECL).
rm -rf ext/google/protobuf/third_party
mkdir -p ext/google/protobuf/third_party/utf8_range
-cp ../third_party/utf8_range/* ext/google/protobuf/third_party/utf8_range
+cp -r ../third_party/utf8_range/* ext/google/protobuf/third_party/utf8_range
echo "Copied utf8_range from ../third_party -> ext/google/protobuf/third_party"
diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl
index c514dcb..95e7124 100644
--- a/protobuf_deps.bzl
+++ b/protobuf_deps.bzl
@@ -63,6 +63,14 @@
urls = ["https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.4.tar.gz"],
)
+ if not native.existing_rule("utf8_range"):
+ _github_archive(
+ name = "utf8_range",
+ repo = "https://github.com/protocolbuffers/utf8_range",
+ commit = "45fbf543fec00020a08650791a37575319a3ea1d",
+ sha256 = "dd93db062025f563068abaa224549e9d341434b5851e959c7853dfa263c96416",
+ )
+
if not native.existing_rule("rules_cc"):
_github_archive(
name = "rules_cc",
diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel
index 5e15d92..d4114b5 100644
--- a/ruby/BUILD.bazel
+++ b/ruby/BUILD.bazel
@@ -71,7 +71,7 @@
name = "protobuf",
srcs = [
":srcs",
- "//third_party/utf8_range:all_files",
+ "@utf8_range//:utf8_range_srcs",
] + select({
":java_ruby": [":protobuf_java"],
"@bazel_tools//src/conditions:darwin": [":protobuf_c_mac"],
diff --git a/ruby/Rakefile b/ruby/Rakefile
index 89a00e1..e6b774c 100644
--- a/ruby/Rakefile
+++ b/ruby/Rakefile
@@ -84,12 +84,16 @@
else
unless ENV['IN_DOCKER'] == 'true'
# We need utf8_range in-tree.
+ if ENV['BAZEL'] == 'true'
+ utf8_root = '../external/utf8_range'
+ else
+ utf8_root = '../third_party/utf8_range'
+ end
FileUtils.mkdir_p("ext/google/protobuf_c/third_party/utf8_range")
- FileUtils.cp("../third_party/utf8_range/utf8_range.h", "ext/google/protobuf_c/third_party/utf8_range")
- FileUtils.cp("../third_party/utf8_range/naive.c", "ext/google/protobuf_c/third_party/utf8_range")
- FileUtils.cp("../third_party/utf8_range/range2-neon.c", "ext/google/protobuf_c/third_party/utf8_range")
- FileUtils.cp("../third_party/utf8_range/range2-sse.c", "ext/google/protobuf_c/third_party/utf8_range")
- FileUtils.cp("../third_party/utf8_range/LICENSE", "ext/google/protobuf_c/third_party/utf8_range")
+ FileUtils.cp(utf8_root+"/utf8_range.h", "ext/google/protobuf_c/third_party/utf8_range")
+ FileUtils.cp(utf8_root+"/naive.c", "ext/google/protobuf_c/third_party/utf8_range")
+ FileUtils.cp(utf8_root+"/range2-neon.c", "ext/google/protobuf_c/third_party/utf8_range")
+ FileUtils.cp(utf8_root+"/range2-sse.c", "ext/google/protobuf_c/third_party/utf8_range")
end
Rake::ExtensionTask.new("protobuf_c", spec) do |ext|
diff --git a/ruby/internal.bzl b/ruby/internal.bzl
index 74a2d10..10ae108 100644
--- a/ruby/internal.bzl
+++ b/ruby/internal.bzl
@@ -23,7 +23,7 @@
":srcs",
":test_ruby_protos",
":tests",
- "//third_party/utf8_range:all_files",
+ "@utf8_range//:utf8_range_srcs",
],
tags = ["manual"],
outs = [extension],
diff --git a/third_party/utf8_range b/third_party/utf8_range
new file mode 160000
index 0000000..45fbf54
--- /dev/null
+++ b/third_party/utf8_range
@@ -0,0 +1 @@
+Subproject commit 45fbf543fec00020a08650791a37575319a3ea1d
diff --git a/third_party/utf8_range/BUILD.bazel b/third_party/utf8_range/BUILD.bazel
deleted file mode 100644
index 693f390..0000000
--- a/third_party/utf8_range/BUILD.bazel
+++ /dev/null
@@ -1,37 +0,0 @@
-load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
-
-# Pulled from: https://github.com/cyb70289/utf8
-
-cc_library(
- name = "utf8_range",
- hdrs = ["utf8_range.h"],
- srcs = [
- "naive.c",
- "range2-neon.c",
- "range2-sse.c",
- ],
- visibility = ["//:__pkg__"],
-)
-
-filegroup(
- name = "all_files",
- srcs = glob([
- "*.h",
- "*.c",
- ]) + [
- "BUILD.bazel",
- "LICENSE",
- ],
- visibility = [
- "//cmake:__pkg__",
- "//php:__pkg__",
- "//ruby:__pkg__",
- ],
-)
-
-pkg_files(
- name = "dist_files",
- srcs = [":all_files"],
- strip_prefix = strip_prefix.from_root(""),
- visibility = ["//pkg:__pkg__"],
-)
diff --git a/third_party/utf8_range/LICENSE b/third_party/utf8_range/LICENSE
deleted file mode 100644
index c1f087e..0000000
--- a/third_party/utf8_range/LICENSE
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2019 Yibo Cai
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/third_party/utf8_range/naive.c b/third_party/utf8_range/naive.c
deleted file mode 100644
index 9b7e5bb..0000000
--- a/third_party/utf8_range/naive.c
+++ /dev/null
@@ -1,92 +0,0 @@
-#include <stdio.h>
-
-/*
- * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
- *
- * Table 3-7. Well-Formed UTF-8 Byte Sequences
- *
- * +--------------------+------------+-------------+------------+-------------+
- * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+0000..U+007F | 00..7F | | | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+0080..U+07FF | C2..DF | 80..BF | | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
- * +--------------------+------------+-------------+------------+-------------+
- * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
- * +--------------------+------------+-------------+------------+-------------+
- */
-
-/* Return 0 - success, >0 - index(1 based) of first error char */
-int utf8_naive(const unsigned char *data, int len)
-{
- int err_pos = 1;
-
- while (len) {
- int bytes;
- const unsigned char byte1 = data[0];
-
- /* 00..7F */
- if (byte1 <= 0x7F) {
- bytes = 1;
- /* C2..DF, 80..BF */
- } else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
- (signed char)data[1] <= (signed char)0xBF) {
- bytes = 2;
- } else if (len >= 3) {
- const unsigned char byte2 = data[1];
-
- /* Is byte2, byte3 between 0x80 ~ 0xBF */
- const int byte2_ok = (signed char)byte2 <= (signed char)0xBF;
- const int byte3_ok = (signed char)data[2] <= (signed char)0xBF;
-
- if (byte2_ok && byte3_ok &&
- /* E0, A0..BF, 80..BF */
- ((byte1 == 0xE0 && byte2 >= 0xA0) ||
- /* E1..EC, 80..BF, 80..BF */
- (byte1 >= 0xE1 && byte1 <= 0xEC) ||
- /* ED, 80..9F, 80..BF */
- (byte1 == 0xED && byte2 <= 0x9F) ||
- /* EE..EF, 80..BF, 80..BF */
- (byte1 >= 0xEE && byte1 <= 0xEF))) {
- bytes = 3;
- } else if (len >= 4) {
- /* Is byte4 between 0x80 ~ 0xBF */
- const int byte4_ok = (signed char)data[3] <= (signed char)0xBF;
-
- if (byte2_ok && byte3_ok && byte4_ok &&
- /* F0, 90..BF, 80..BF, 80..BF */
- ((byte1 == 0xF0 && byte2 >= 0x90) ||
- /* F1..F3, 80..BF, 80..BF, 80..BF */
- (byte1 >= 0xF1 && byte1 <= 0xF3) ||
- /* F4, 80..8F, 80..BF, 80..BF */
- (byte1 == 0xF4 && byte2 <= 0x8F))) {
- bytes = 4;
- } else {
- return err_pos;
- }
- } else {
- return err_pos;
- }
- } else {
- return err_pos;
- }
-
- len -= bytes;
- err_pos += bytes;
- data += bytes;
- }
-
- return 0;
-}
diff --git a/third_party/utf8_range/range2-neon.c b/third_party/utf8_range/range2-neon.c
deleted file mode 100644
index 6833965..0000000
--- a/third_party/utf8_range/range2-neon.c
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Process 2x16 bytes in each iteration.
- * Comments removed for brevity. See range-neon.c for details.
- */
-#if defined(__aarch64__) && defined(__ARM_NEON)
-
-#include <stdio.h>
-#include <stdint.h>
-#include <arm_neon.h>
-
-int utf8_naive(const unsigned char *data, int len);
-
-static const uint8_t _first_len_tbl[] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
-};
-
-static const uint8_t _first_range_tbl[] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
-};
-
-static const uint8_t _range_min_tbl[] = {
- 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
- 0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
-};
-static const uint8_t _range_max_tbl[] = {
- 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
- 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-};
-
-static const uint8_t _range_adjust_tbl[] = {
- 2, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0,
-};
-
-/* Return 0 on success, -1 on error */
-int utf8_range2(const unsigned char *data, int len)
-{
- if (len >= 32) {
- uint8x16_t prev_input = vdupq_n_u8(0);
- uint8x16_t prev_first_len = vdupq_n_u8(0);
-
- const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl);
- const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl);
- const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl);
- const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl);
- const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl);
-
- const uint8x16_t const_1 = vdupq_n_u8(1);
- const uint8x16_t const_2 = vdupq_n_u8(2);
- const uint8x16_t const_e0 = vdupq_n_u8(0xE0);
-
- uint8x16_t error1 = vdupq_n_u8(0);
- uint8x16_t error2 = vdupq_n_u8(0);
- uint8x16_t error3 = vdupq_n_u8(0);
- uint8x16_t error4 = vdupq_n_u8(0);
-
- while (len >= 32) {
- /******************* two blocks interleaved **********************/
-
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 8)
- /* gcc doesn't support vldq1_u8_x2 until version 8 */
- const uint8x16_t input_a = vld1q_u8(data);
- const uint8x16_t input_b = vld1q_u8(data + 16);
-#else
- /* Forces a double load on Clang */
- const uint8x16x2_t input_pair = vld1q_u8_x2(data);
- const uint8x16_t input_a = input_pair.val[0];
- const uint8x16_t input_b = input_pair.val[1];
-#endif
-
- const uint8x16_t high_nibbles_a = vshrq_n_u8(input_a, 4);
- const uint8x16_t high_nibbles_b = vshrq_n_u8(input_b, 4);
-
- const uint8x16_t first_len_a =
- vqtbl1q_u8(first_len_tbl, high_nibbles_a);
- const uint8x16_t first_len_b =
- vqtbl1q_u8(first_len_tbl, high_nibbles_b);
-
- uint8x16_t range_a = vqtbl1q_u8(first_range_tbl, high_nibbles_a);
- uint8x16_t range_b = vqtbl1q_u8(first_range_tbl, high_nibbles_b);
-
- range_a =
- vorrq_u8(range_a, vextq_u8(prev_first_len, first_len_a, 15));
- range_b =
- vorrq_u8(range_b, vextq_u8(first_len_a, first_len_b, 15));
-
- uint8x16_t tmp1_a, tmp2_a, tmp1_b, tmp2_b;
- tmp1_a = vextq_u8(prev_first_len, first_len_a, 14);
- tmp1_a = vqsubq_u8(tmp1_a, const_1);
- range_a = vorrq_u8(range_a, tmp1_a);
-
- tmp1_b = vextq_u8(first_len_a, first_len_b, 14);
- tmp1_b = vqsubq_u8(tmp1_b, const_1);
- range_b = vorrq_u8(range_b, tmp1_b);
-
- tmp2_a = vextq_u8(prev_first_len, first_len_a, 13);
- tmp2_a = vqsubq_u8(tmp2_a, const_2);
- range_a = vorrq_u8(range_a, tmp2_a);
-
- tmp2_b = vextq_u8(first_len_a, first_len_b, 13);
- tmp2_b = vqsubq_u8(tmp2_b, const_2);
- range_b = vorrq_u8(range_b, tmp2_b);
-
- uint8x16_t shift1_a = vextq_u8(prev_input, input_a, 15);
- uint8x16_t pos_a = vsubq_u8(shift1_a, const_e0);
- range_a = vaddq_u8(range_a, vqtbl2q_u8(range_adjust_tbl, pos_a));
-
- uint8x16_t shift1_b = vextq_u8(input_a, input_b, 15);
- uint8x16_t pos_b = vsubq_u8(shift1_b, const_e0);
- range_b = vaddq_u8(range_b, vqtbl2q_u8(range_adjust_tbl, pos_b));
-
- uint8x16_t minv_a = vqtbl1q_u8(range_min_tbl, range_a);
- uint8x16_t maxv_a = vqtbl1q_u8(range_max_tbl, range_a);
-
- uint8x16_t minv_b = vqtbl1q_u8(range_min_tbl, range_b);
- uint8x16_t maxv_b = vqtbl1q_u8(range_max_tbl, range_b);
-
- error1 = vorrq_u8(error1, vcltq_u8(input_a, minv_a));
- error2 = vorrq_u8(error2, vcgtq_u8(input_a, maxv_a));
-
- error3 = vorrq_u8(error3, vcltq_u8(input_b, minv_b));
- error4 = vorrq_u8(error4, vcgtq_u8(input_b, maxv_b));
-
- /************************ next iteration *************************/
- prev_input = input_b;
- prev_first_len = first_len_b;
-
- data += 32;
- len -= 32;
- }
- error1 = vorrq_u8(error1, error2);
- error1 = vorrq_u8(error1, error3);
- error1 = vorrq_u8(error1, error4);
-
- if (vmaxvq_u8(error1))
- return -1;
-
- uint32_t token4;
- vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3);
-
- const int8_t *token = (const int8_t *)&token4;
- int lookahead = 0;
- if (token[3] > (int8_t)0xBF)
- lookahead = 1;
- else if (token[2] > (int8_t)0xBF)
- lookahead = 2;
- else if (token[1] > (int8_t)0xBF)
- lookahead = 3;
-
- data -= lookahead;
- len += lookahead;
- }
-
- return utf8_naive(data, len);
-}
-
-#endif
diff --git a/third_party/utf8_range/range2-sse.c b/third_party/utf8_range/range2-sse.c
deleted file mode 100644
index f3deb86..0000000
--- a/third_party/utf8_range/range2-sse.c
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Process 2x16 bytes in each iteration.
- * Comments removed for brevity. See range-sse.c for details.
- */
-#ifdef __SSE4_1__
-
-#include <stdio.h>
-#include <stdint.h>
-#include <x86intrin.h>
-
-int utf8_naive(const unsigned char *data, int len);
-
-static const int8_t _first_len_tbl[] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
-};
-
-static const int8_t _first_range_tbl[] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
-};
-
-static const int8_t _range_min_tbl[] = {
- 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
- 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
-};
-static const int8_t _range_max_tbl[] = {
- 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
- 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
-};
-
-static const int8_t _df_ee_tbl[] = {
- 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
-};
-static const int8_t _ef_fe_tbl[] = {
- 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-};
-
-/* Return 0 on success, -1 on error */
-int utf8_range2(const unsigned char *data, int len)
-{
- if (len >= 32) {
- __m128i prev_input = _mm_set1_epi8(0);
- __m128i prev_first_len = _mm_set1_epi8(0);
-
- const __m128i first_len_tbl =
- _mm_loadu_si128((const __m128i *)_first_len_tbl);
- const __m128i first_range_tbl =
- _mm_loadu_si128((const __m128i *)_first_range_tbl);
- const __m128i range_min_tbl =
- _mm_loadu_si128((const __m128i *)_range_min_tbl);
- const __m128i range_max_tbl =
- _mm_loadu_si128((const __m128i *)_range_max_tbl);
- const __m128i df_ee_tbl =
- _mm_loadu_si128((const __m128i *)_df_ee_tbl);
- const __m128i ef_fe_tbl =
- _mm_loadu_si128((const __m128i *)_ef_fe_tbl);
-
- __m128i error = _mm_set1_epi8(0);
-
- while (len >= 32) {
- /***************************** block 1 ****************************/
- const __m128i input_a = _mm_loadu_si128((const __m128i *)data);
-
- __m128i high_nibbles =
- _mm_and_si128(_mm_srli_epi16(input_a, 4), _mm_set1_epi8(0x0F));
-
- __m128i first_len_a = _mm_shuffle_epi8(first_len_tbl, high_nibbles);
-
- __m128i range_a = _mm_shuffle_epi8(first_range_tbl, high_nibbles);
-
- range_a = _mm_or_si128(
- range_a, _mm_alignr_epi8(first_len_a, prev_first_len, 15));
-
- __m128i tmp;
- tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 14);
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1));
- range_a = _mm_or_si128(range_a, tmp);
-
- tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 13);
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2));
- range_a = _mm_or_si128(range_a, tmp);
-
- __m128i shift1, pos, range2;
- shift1 = _mm_alignr_epi8(input_a, prev_input, 15);
- pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
- tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0));
- range2 = _mm_shuffle_epi8(df_ee_tbl, tmp);
- tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70));
- range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp));
-
- range_a = _mm_add_epi8(range_a, range2);
-
- __m128i minv = _mm_shuffle_epi8(range_min_tbl, range_a);
- __m128i maxv = _mm_shuffle_epi8(range_max_tbl, range_a);
-
- tmp = _mm_or_si128(
- _mm_cmplt_epi8(input_a, minv),
- _mm_cmpgt_epi8(input_a, maxv)
- );
- error = _mm_or_si128(error, tmp);
-
- /***************************** block 2 ****************************/
- const __m128i input_b = _mm_loadu_si128((const __m128i *)(data+16));
-
- high_nibbles =
- _mm_and_si128(_mm_srli_epi16(input_b, 4), _mm_set1_epi8(0x0F));
-
- __m128i first_len_b = _mm_shuffle_epi8(first_len_tbl, high_nibbles);
-
- __m128i range_b = _mm_shuffle_epi8(first_range_tbl, high_nibbles);
-
- range_b = _mm_or_si128(
- range_b, _mm_alignr_epi8(first_len_b, first_len_a, 15));
-
-
- tmp = _mm_alignr_epi8(first_len_b, first_len_a, 14);
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1));
- range_b = _mm_or_si128(range_b, tmp);
-
- tmp = _mm_alignr_epi8(first_len_b, first_len_a, 13);
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2));
- range_b = _mm_or_si128(range_b, tmp);
-
- shift1 = _mm_alignr_epi8(input_b, input_a, 15);
- pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
- tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0));
- range2 = _mm_shuffle_epi8(df_ee_tbl, tmp);
- tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70));
- range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp));
-
- range_b = _mm_add_epi8(range_b, range2);
-
- minv = _mm_shuffle_epi8(range_min_tbl, range_b);
- maxv = _mm_shuffle_epi8(range_max_tbl, range_b);
-
-
- tmp = _mm_or_si128(
- _mm_cmplt_epi8(input_b, minv),
- _mm_cmpgt_epi8(input_b, maxv)
- );
- error = _mm_or_si128(error, tmp);
-
- /************************ next iteration **************************/
- prev_input = input_b;
- prev_first_len = first_len_b;
-
- data += 32;
- len -= 32;
- }
-
- if (!_mm_testz_si128(error, error))
- return -1;
-
- int32_t token4 = _mm_extract_epi32(prev_input, 3);
- const int8_t *token = (const int8_t *)&token4;
- int lookahead = 0;
- if (token[3] > (int8_t)0xBF)
- lookahead = 1;
- else if (token[2] > (int8_t)0xBF)
- lookahead = 2;
- else if (token[1] > (int8_t)0xBF)
- lookahead = 3;
-
- data -= lookahead;
- len += lookahead;
- }
-
- return utf8_naive(data, len);
-}
-
-#endif
diff --git a/third_party/utf8_range/utf8_range.h b/third_party/utf8_range/utf8_range.h
deleted file mode 100644
index 23b32a1..0000000
--- a/third_party/utf8_range/utf8_range.h
+++ /dev/null
@@ -1,9 +0,0 @@
-
-#if ((defined(__ARM_NEON) && defined(__aarch64__)) || defined(__SSE4_1__)) && !defined(TRUFFLERUBY)
-int utf8_range2(const unsigned char* data, int len);
-#else
-int utf8_naive(const unsigned char* data, int len);
-static inline int utf8_range2(const unsigned char* data, int len) {
- return utf8_naive(data, len);
-}
-#endif
