feat: add SplitWords

strutil/string.go

@@ -287,3 +287,45 @@ func Substring(s string, offset int, length uint) string {
 
 	return strings.Replace(str, "\x00", "", -1)
 }
+
+// SplitWords splits a string into words, word only contains alphabetic characters.
+// Play: todo
+func SplitWords(s string) []string {
+	var word string
+	var words []string
+	var r rune
+	var size, pos int
+
+	isWord := false
+
+	for len(s) > 0 {
+		r, size = utf8.DecodeRuneInString(s)
+
+		switch {
+		case isLetter(r):
+			if !isWord {
+				isWord = true
+				word = s
+				pos = 0
+			}
+
+		case isWord && (r == '\'' || r == '-'):
+			// is word
+
+		default:
+			if isWord {
+				isWord = false
+				words = append(words, word[:pos])
+			}
+		}
+
+		pos += size
+		s = s[size:]
+	}
+
+	if isWord {
+		words = append(words, word[:pos])
+	}
+
+	return words
+}

strutil/string_example_test.go

@@ -388,3 +388,28 @@ func ExampleSubstring() {
 	// de
 	// 你好
 }
+
+func ExampleSplitWords() {
+
+	result1 := SplitWords("a word")
+	result2 := SplitWords("I'am a programmer")
+	result3 := SplitWords("Bonjour, je suis programmeur")
+	result4 := SplitWords("a -b-c' 'd'e")
+	result5 := SplitWords("你好，我是一名码农")
+	result6 := SplitWords("こんにちは，私はプログラマーです")
+
+	fmt.Println(result1)
+	fmt.Println(result2)
+	fmt.Println(result3)
+	fmt.Println(result4)
+	fmt.Println(result5)
+	fmt.Println(result6)
+
+	// Output:
+	// [a word]
+	// [I'am a programmer]
+	// [Bonjour je suis programmeur]
+	// [a b-c' d'e]
+	// []
+	// []
+}

strutil/string_internal.go

@@ -135,3 +135,36 @@ func padAtPosition(str string, length int, padStr string, position int) string {
 
 	return leftPad + str + rightPad
 }
+
+// isLetter checks r is a letter but not CJK character.
+func isLetter(r rune) bool {
+	if !unicode.IsLetter(r) {
+		return false
+	}
+
+	switch {
+	// cjk char: /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9f]/
+
+	// hiragana and katakana (Japanese only)
+	case r >= '\u3034' && r < '\u30ff':
+		return false
+
+	// CJK unified ideographs extension A (Chinese, Japanese, and Korean)
+	case r >= '\u3400' && r < '\u4dbf':
+		return false
+
+	// CJK unified ideographs (Chinese, Japanese, and Korean)
+	case r >= '\u4e00' && r < '\u9fff':
+		return false
+
+	// CJK compatibility ideographs (Chinese, Japanese, and Korean)
+	case r >= '\uf900' && r < '\ufaff':
+		return false
+
+	// half-width katakana (Japanese only)
+	case r >= '\uff66' && r < '\uff9f':
+		return false
+	}
+
+	return true
+}

strutil/string_test.go

@@ -308,3 +308,20 @@ func TestSubstring(t *testing.T) {
 	assert.Equal("de", Substring("abcde", -2, 3))
 	assert.Equal("你好", Substring("你好，欢迎你", 0, 2))
 }
+
+func TestSplitWords(t *testing.T) {
+	assert := internal.NewAssert(t, "TestSplitWords")
+
+	cases := map[string][]string{
+		"a word":                       {"a", "word"},
+		"I'am a programmer":            {"I'am", "a", "programmer"},
+		"Bonjour, je suis programmeur": {"Bonjour", "je", "suis", "programmeur"},
+		"a -b-c' 'd'e":                 {"a", "b-c'", "d'e"},
+		"你好，我是一名码农":                    nil,
+		"こんにちは，私はプログラマーです": nil,
+	}
+
+	for k, v := range cases {
+		assert.Equal(v, SplitWords(k))
+	}
+}