Reedbeta · August 7, 2020 07:06 · Aug 7, 2020
diff --git a/itertokens.h b/itertokens.h
@@ -0,0 +1,56 @@
+#pragma once
+
+// String tokenizing iterator
+// by Nathan Reed, 2020-08-06. Licensed CC0 https://creativecommons.org/publicdomain/zero/1.0/
+//
+// Use it like:
+//
+// for (auto token : IterTokens(" Bird \t\tFish Dog   Cat "))
+// {
+//     // token is a std::string_view pointing into the original string
+//     std::cout << "Token: " << token << "\n";
+// }
+//
+// The input string is not modified.
+// Both the input string and the delimiter string must remain valid at least till the end of the loop.
+// (This could be made more sophisticated so that it could take other string types as input,
+// auto-capture temporaries, etc., but I haven't done that here.)
+// Godbolt example: https://godbolt.org/z/8PnjrW
+
+constexpr auto IterTokens(const char* str, const char* delimiters = "\t ")
+{
+    struct iterator
+    {
+        const char* tokenStart;
+        const char* tokenEnd;
+        const char* delimiters;
+        bool operator != (const iterator & other) const { return tokenEnd != other.tokenEnd; }
+        bool isDelimiter(char c) { return strchr(delimiters, c) != nullptr; }
+        void next()
+        {
+            if (!tokenEnd) return;
+
+            // Find the beginning of the next token
+            tokenStart = tokenEnd;
+            while (*tokenStart != 0 && isDelimiter(*tokenStart)) ++tokenStart;
+            if (*tokenStart == 0) { tokenEnd = nullptr; return; }
+
+            // Find the end of the token
+            tokenEnd = tokenStart;
+            while (*tokenEnd != 0 && !isDelimiter(*tokenEnd)) ++tokenEnd;
+        }
+        iterator& operator ++ () { next(); return *this; }
+        std::string_view operator * () const
+        {
+            return tokenEnd ? std::string_view(tokenStart, size_t(tokenEnd - tokenStart)) : std::string_view();
+        }
+    };
+    struct iterable
+    {
+        const char* str;
+        const char* delimiters;
+        auto begin() { return ++iterator{str, str, delimiters}; }
+        auto end() { return iterator{nullptr, nullptr, nullptr}; }
+    };
+    return iterable{str, delimiters};
+}