1
0
mirror of https://github.com/duke-git/lancet.git synced 2026-03-01 00:35:28 +08:00

Compare commits

...

5 Commits

Author SHA1 Message Date
dudaodong 036847577d release v2.3.0 2024-03-05 14:57:16 +08:00
dudaodong d21edd1cde doc: add doc for retry package 2024-03-05 14:38:42 +08:00
dudaodong c58c50327c doc: add doc for ChunkRead and ParallelChunkRead 2024-03-05 11:42:03 +08:00
dudaodong 9bfdc686f8 doc: update contribution_guide file 2024-03-05 10:47:51 +08:00
dudaodong 5ca8f6ef6f doc: update doc 2024-03-05 10:45:40 +08:00
17 changed files with 940 additions and 153 deletions
+1 -1
View File
@@ -30,7 +30,7 @@ We are excited that you are interested in contributing to lancet. Before submitt
- Before submitting a PR, please execute the unit test command: `go test -v ./...` to ensure that all unit test tasks should pass.
- Make sure PRs are created to `rc` branch instead of `master` branch.
- Make sure PRs are created to `rc` branch instead of other branch.
- If your PR fixes a bug, please provide a description about the related bug.
+16 -2
View File
@@ -4,7 +4,7 @@
<br/>
![Go version](https://img.shields.io/badge/go-%3E%3Dv1.18-9cf)
[![Release](https://img.shields.io/badge/release-2.2.9-green.svg)](https://github.com/duke-git/lancet/releases)
[![Release](https://img.shields.io/badge/release-2.3.0-green.svg)](https://github.com/duke-git/lancet/releases)
[![GoDoc](https://godoc.org/github.com/duke-git/lancet/v2?status.svg)](https://pkg.go.dev/github.com/duke-git/lancet/v2)
[![Go Report Card](https://goreportcard.com/badge/github.com/duke-git/lancet/v2)](https://goreportcard.com/report/github.com/duke-git/lancet/v2)
[![test](https://github.com/duke-git/lancet/actions/workflows/codecov.yml/badge.svg?branch=main&event=push)](https://github.com/duke-git/lancet/actions/workflows/codecov.yml)
@@ -38,7 +38,7 @@
go get github.com/duke-git/lancet/v2 // will install latest version of v2.x.x
```
2. <b>For users who use version below go1.18, you should install v1.x.x. The latest of v1.x.x is v1.4.2. </b>
2. <b>For users who use version below go1.18, you should install v1.x.x. The latest of v1.x.x is v1.4.3. </b>
```go
go get github.com/duke-git/lancet // below go1.18, install latest version of v1.x.x
@@ -729,6 +729,10 @@ import "github.com/duke-git/lancet/v2/fileutil"
[[play](https://go.dev/play/p/GhLS6d8lH_g)]
- **<big>ReadFile</big>** : read file or url.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/fileutil.md#ReadFile)]
- **<big>ChunkRead</big>** : reads a block from the file at the specified offset and returns all lines within the block.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/fileutil.md#ChunkRead)]
- **<big>ParallelChunkRead</big>** : reads the file in parallel and send each chunk of lines to the specified channel.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/fileutil.md#ParallelChunkRead)]
<h3 id="formatter"> 10. Formatter contains some functions for data formatting. &nbsp; &nbsp; &nbsp; &nbsp;<a href="#index">index</a></h3>
@@ -1172,6 +1176,16 @@ import "github.com/duke-git/lancet/v2/retry"
- **<big>RetryTimes</big>** : set times of retry.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/retry.md#RetryTimes)]
[[play](https://go.dev/play/p/ssfVeU2SwLO)]
- **<big>BackoffStrategy</big>** : An interface that defines a method for calculating backoff intervals.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/retry.md#BackoffStrategy)]
- **<big>RetryWithCustomBackoff</big>** : set abitary custom backoff strategy.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/retry.md#RetryWithCustomBackoff)]
- **<big>RetryWithLinearBackoff</big>** : set linear strategy backoff.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/retry.md#RetryWithLinearBackoff)]
- **<big>RetryWithExponentialWithJitterBackoff</big>** : set exponential strategy backoff.
[[doc](https://github.com/duke-git/lancet/blob/main/docs/en/api/packages/retry.md#RetryWithExponentialWithJitterBackoff)]
<h3 id="slice"> 18. Slice contains some functions to manipulate slice. &nbsp; &nbsp; &nbsp; &nbsp;<a href="#index">index</a></h3>
+18 -2
View File
@@ -4,7 +4,7 @@
<br/>
![Go version](https://img.shields.io/badge/go-%3E%3Dv1.18-9cf)
[![Release](https://img.shields.io/badge/release-2.2.9-green.svg)](https://github.com/duke-git/lancet/releases)
[![Release](https://img.shields.io/badge/release-2.3.0-green.svg)](https://github.com/duke-git/lancet/releases)
[![GoDoc](https://godoc.org/github.com/duke-git/lancet/v2?status.svg)](https://pkg.go.dev/github.com/duke-git/lancet/v2)
[![Go Report Card](https://goreportcard.com/badge/github.com/duke-git/lancet/v2)](https://goreportcard.com/report/github.com/duke-git/lancet/v2)
[![test](https://github.com/duke-git/lancet/actions/workflows/codecov.yml/badge.svg?branch=main&event=push)](https://github.com/duke-git/lancet/actions/workflows/codecov.yml)
@@ -37,7 +37,7 @@
go get github.com/duke-git/lancet/v2 //安装v2最新版本v2.x.x
```
2. <b>使用 go1.18 以下版本的用户,必须安装 v1.x.x。目前最新的 v1 版本是 v1.4.2。</b>
2. <b>使用 go1.18 以下版本的用户,必须安装 v1.x.x。目前最新的 v1 版本是 v1.4.3。</b>
```go
go get github.com/duke-git/lancet// 使用go1.18以下版本, 必须安装v1.x.x版本
@@ -728,6 +728,12 @@ import "github.com/duke-git/lancet/v2/fileutil"
[[play](https://go.dev/play/p/GhLS6d8lH_g)]
- **<big>ReadFile</big>** : 读取文件或者URL。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/fileutil.md#ReadFile)]
- **<big>ChunkRead</big>** : 从文件的指定偏移读取块并返回块内所有行。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/fileutil.md#ChunkRead)]
- **<big>ParallelChunkRead</big>** : 并行读取文件并将每个块的行发送到指定通道。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/fileutil.md#ParallelChunkRead)]
<h3 id="formatter"> 10. formatter 格式化器包含一些数据格式化处理方法。&nbsp; &nbsp; &nbsp; &nbsp;<a href="#index">回到目录</a></h3>
@@ -1170,6 +1176,16 @@ import "github.com/duke-git/lancet/v2/retry"
- **<big>RetryTimes</big>** : 设置重试次数,默认 5。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/retry.md#RetryTimes)]
[[play](https://go.dev/play/p/ssfVeU2SwLO)]
- **<big>BackoffStrategy</big>** : 定义计算退避间隔的方法的接口。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/retry.md#BackoffStrategy)]
- **<big>RetryWithCustomBackoff</big>** : 设置自定义退避策略。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/retry.md#RetryWithCustomBackoff)]
- **<big>RetryWithLinearBackoff</big>** : 设置线性策略退避。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/retry.md#RetryWithLinearBackoff)]
- **<big>RetryWithExponentialWithJitterBackoff</big>** : 设置指数策略退避。
[[doc](https://github.com/duke-git/lancet/blob/main/docs/api/packages/retry.md#RetryWithExponentialWithJitterBackoff)]
<h3 id="slice"> 18. slice 包含操作切片的方法集合。&nbsp; &nbsp; &nbsp; &nbsp; <a href="#index">回到目录</a></h3>
+116
View File
@@ -50,6 +50,8 @@ import (
- [WriteStringToFile](#WriteStringToFile)
- [WriteBytesToFile](#WriteBytesToFile)
- [ReadFile](#ReadFile)
- [ChunkRead](#ChunkRead)
- [ParallelChunkRead](#ParallelChunkRead)
<div STYLE="page-break-after: always;"></div>
@@ -955,9 +957,123 @@ func main() {
if err != nil {
return
}
fmt.Println(string(dat))
// Output:
// User-agent: *
// Disallow: /deny
}
```
### <span id="ChunkRead">ChunkRead</span>
<p>从文件的指定偏移读取块并返回块内所有行。</p>
<b>函数签名:</b>
```go
func ChunkRead(file *os.File, offset int64, size int, bufPool *sync.Pool) ([]string, error)
```
<b>示例:</b>
```go
package main
import (
"fmt"
"github.com/duke-git/lancet/v2/fileutil"
)
func main() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100
// test1.csv file content:
// Lili,22,female
// Jim,21,male
filePath := "./testdata/test1.csv" // 替换为你的文件路径
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, 0, defaultChunkSizeMB*mb)
},
}
lines, err := fileutil.ChunkRead(f, 0, 100, &bufPool)
if err != nil {
return
}
fmt.Println(lines[0])
fmt.Println(lines[1])
// Output:
// Lili,22,female
// Jim,21,male
}
```
### <span id="ParallelChunkRead">ParallelChunkRead</span>
<p>并行读取文件并将每个块的行发送到指定通道。</p>
<b>函数签名:</b>
```go
// filePath:文件路径
// chunkSizeMB: 分块的大小(单位MB,设置为0时使用默认100MB),设置过大反而不利,视情调整
// maxGoroutine: 并发读取分块的数量,设置为0时使用CPU核心数
// linesCh: 用于接收返回结果的通道。
func ParallelChunkRead(filePath string, linesCh chan<- []string, chunkSizeMB, maxGoroutine int) error
```
<b>示例:</b>
```go
package main
import (
"fmt"
"github.com/duke-git/lancet/v2/fileutil"
)
func main() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100 // 默认值
numParsers := runtime.NumCPU()
linesCh := make(chan []string, numParsers)
// test1.csv file content:
// Lili,22,female
// Jim,21,male
filePath := "./testdata/test1.csv"
go fileutil.ParallelChunkRead(filePath, linesCh, defaultChunkSizeMB, numParsers)
var totalLines int
for lines := range linesCh {
totalLines += len(lines)
for _, line := range lines {
fmt.Println(line)
}
}
fmt.Println(totalLines)
// Output:
// Lili,22,female
// Jim,21,male
// 2
}
```
+59 -59
View File
@@ -519,18 +519,18 @@ import (
)
func main() {
result1 := mathutil.CeilToFloat(3.14159, 1)
result2 := mathutil.CeilToFloat(3.14159, 2)
result3 := mathutil.CeilToFloat(5, 4)
result1 := mathutil.CeilToFloat(3.14159, 1)
result2 := mathutil.CeilToFloat(3.14159, 2)
result3 := mathutil.CeilToFloat(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.2
// 3.15
// 5
// Output:
// 3.2
// 3.15
// 5
}
```
@@ -555,18 +555,18 @@ import (
)
func main() {
result1 := mathutil.CeilToString(3.14159, 1)
result2 := mathutil.CeilToString(3.14159, 2)
result3 := mathutil.CeilToString(5, 4)
result1 := mathutil.CeilToString(3.14159, 1)
result2 := mathutil.CeilToString(3.14159, 2)
result3 := mathutil.CeilToString(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.2
// 3.15
// 5.0000
// Output:
// 3.2
// 3.15
// 5.0000
}
```
@@ -591,18 +591,18 @@ import (
)
func main() {
result1 := mathutil.FloorToFloat(3.14159, 1)
result2 := mathutil.FloorToFloat(3.14159, 2)
result3 := mathutil.FloorToFloat(5, 4)
result1 := mathutil.FloorToFloat(3.14159, 1)
result2 := mathutil.FloorToFloat(3.14159, 2)
result3 := mathutil.FloorToFloat(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.1
// 3.14
// 5
// Output:
// 3.1
// 3.14
// 5
}
```
@@ -627,18 +627,18 @@ import (
)
func main() {
result1 := mathutil.FloorToString(3.14159, 1)
result2 := mathutil.FloorToString(3.14159, 2)
result3 := mathutil.FloorToString(5, 4)
result1 := mathutil.FloorToString(3.14159, 1)
result2 := mathutil.FloorToString(3.14159, 2)
result3 := mathutil.FloorToString(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.1
// 3.14
// 5.0000
// Output:
// 3.1
// 3.14
// 5.0000
}
```
@@ -1114,17 +1114,17 @@ import (
func main() {
result1 := Abs(-1)
result2 := Abs(-0.1)
result3 := Abs(float32(0.2))
result2 := Abs(-0.1)
result3 := Abs(float32(0.2))
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 1
// 0.1
// 0.2
// Output:
// 1
// 0.1
// 0.2
}
```
@@ -1149,16 +1149,16 @@ import (
)
func main() {
result1 := mathutil.Div(9, 4)
result2 := mathutil.Div(1, 2)
result3 := mathutil.Div(0, 666)
result1 := mathutil.Div(9, 4)
result2 := mathutil.Div(1, 2)
result3 := mathutil.Div(0, 666)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 2.25
// 0.5
// 0
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 2.25
// 0.5
// 0
}
```
+202
View File
@@ -27,6 +27,10 @@ import (
- [RetryFunc](#RetryFunc)
- [RetryDuration](#RetryDuration)
- [RetryTimes](#RetryTimes)
- [BackoffStrategy](#BackoffStrategy)
- [RetryWithCustomBackoff](#RetryWithCustomBackoff)
- [RetryWithLinearBackoff](#RetryWithLinearBackoff)
- [RetryWithExponentialWithJitterBackoff](#RetryWithExponentialWithJitterBackoff)
<div STYLE="page-break-after: always;"></div>
@@ -260,3 +264,201 @@ func main() {
// 3
}
```
### <span id="BackoffStrategy">BackoffStrategy</span>
<p>定义计算退避间隔的方法的接口。</p>
<b>函数签名:</b>
```go
// BackoffStrategy is an interface that defines a method for calculating backoff intervals.
type BackoffStrategy interface {
// CalculateInterval returns the time.Duration after which the next retry attempt should be made.
CalculateInterval() time.Duration
}
```
<b>示例:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
type ExampleCustomBackoffStrategy struct {
interval time.Duration
}
func (c *ExampleCustomBackoffStrategy) CalculateInterval() time.Duration {
return c.interval + 1
}
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry,Retry(increaseNumber, retry.RetryWithCustomBackoff(&示例CustomBackoffStrategy{interval: time.Microsecond * 50}))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithCustomBackoff">RetryWithCustomBackoff</span>
<p>设置自定义退避策略。</p>
<b>函数签名:</b>
```go
func RetryWithCustomBackoff(backoffStrategy BackoffStrategy) Option
```
<b>示例:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
type ExampleCustomBackoffStrategy struct {
interval time.Duration
}
func (c *ExampleCustomBackoffStrategy) CalculateInterval() time.Duration {
return c.interval + 1
}
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry,Retry(increaseNumber, retry.RetryWithCustomBackoff(&示例CustomBackoffStrategy{interval: time.Microsecond * 50}))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithLinearBackoff">RetryWithLinearBackoff</span>
<p>设置线性策略退避。</p>
<b>函数签名:</b>
```go
func RetryWithLinearBackoff(interval time.Duration) Option
```
<b>示例:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry.Retry(increaseNumber, retry.RetryWithLinearBackoff(time.Microsecond*50))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithExponentialWithJitterBackoff">RetryWithExponentialWithJitterBackoff</span>
<p>设置指数策略退避。</p>
<b>函数签名:</b>
```go
func RetryWithExponentialWithJitterBackoff(interval time.Duration, base uint64, maxJitter time.Duration) Option
```
<b>示例:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry.Retry(increaseNumber, retry.RetryWithExponentialWithJitterBackoff(time.Microsecond*50, 2, time.Microsecond*25))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
+114
View File
@@ -50,6 +50,8 @@ import (
- [WriteStringToFile](#WriteStringToFile)
- [WriteBytesToFile](#WriteBytesToFile)
- [ReadFile](#ReadFile)
- [ChunkRead](#ChunkRead)
- [ParallelChunkRead](#ParallelChunkRead)
<div STYLE="page-break-after: always;"></div>
@@ -961,3 +963,115 @@ func main() {
// Disallow: /deny
}
```
### <span id="ChunkRead">ChunkRead</span>
<p>reads a block from the file at the specified offset and returns all lines within the block.</p>
<b>Signature :</b>
```go
func ChunkRead(file *os.File, offset int64, size int, bufPool *sync.Pool) ([]string, error)
```
<b>Example:</b>
```go
package main
import (
"fmt"
"github.com/duke-git/lancet/v2/fileutil"
)
func main() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100
// test1.csv file content:
// Lili,22,female
// Jim,21,male
filePath := "./testdata/test1.csv"
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, 0, defaultChunkSizeMB*mb)
},
}
lines, err := fileutil.ChunkRead(f, 0, 100, &bufPool)
if err != nil {
return
}
fmt.Println(lines[0])
fmt.Println(lines[1])
// Output:
// Lili,22,female
// Jim,21,male
}
```
### <span id="ParallelChunkRead">ParallelChunkRead</span>
<p>Reads the file in parallel and send each chunk of lines to the specified channel.</p>
<b>Signature :</b>
```go
// filePath: file path.
// chunkSizeMB: The size of the block (in MB, the default is 100MB when set to 0). Setting it too large will be detrimental. Adjust it as appropriate.
// maxGoroutine: The number of concurrent read chunks, the number of CPU cores used when set to 0.
// linesCh: The channel used to receive the returned results.
func ParallelChunkRead(filePath string, linesCh chan<- []string, chunkSizeMB, maxGoroutine int) error
```
<b>Example:</b>
```go
package main
import (
"fmt"
"github.com/duke-git/lancet/v2/fileutil"
)
func main() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100 // 默认值
numParsers := runtime.NumCPU()
linesCh := make(chan []string, numParsers)
// test1.csv file content:
// Lili,22,female
// Jim,21,male
filePath := "./testdata/test1.csv"
go fileutil.ParallelChunkRead(filePath, linesCh, defaultChunkSizeMB, numParsers)
var totalLines int
for lines := range linesCh {
totalLines += len(lines)
for _, line := range lines {
fmt.Println(line)
}
}
fmt.Println(totalLines)
// Output:
// Lili,22,female
// Jim,21,male
// 2
}
```
+59 -59
View File
@@ -519,18 +519,18 @@ import (
)
func main() {
result1 := mathutil.CeilToFloat(3.14159, 1)
result2 := mathutil.CeilToFloat(3.14159, 2)
result3 := mathutil.CeilToFloat(5, 4)
result1 := mathutil.CeilToFloat(3.14159, 1)
result2 := mathutil.CeilToFloat(3.14159, 2)
result3 := mathutil.CeilToFloat(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.2
// 3.15
// 5
// Output:
// 3.2
// 3.15
// 5
}
```
@@ -555,18 +555,18 @@ import (
)
func main() {
result1 := mathutil.CeilToString(3.14159, 1)
result2 := mathutil.CeilToString(3.14159, 2)
result3 := mathutil.CeilToString(5, 4)
result1 := mathutil.CeilToString(3.14159, 1)
result2 := mathutil.CeilToString(3.14159, 2)
result3 := mathutil.CeilToString(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.2
// 3.15
// 5.0000
// Output:
// 3.2
// 3.15
// 5.0000
}
```
@@ -591,18 +591,18 @@ import (
)
func main() {
result1 := mathutil.FloorToFloat(3.14159, 1)
result2 := mathutil.FloorToFloat(3.14159, 2)
result3 := mathutil.FloorToFloat(5, 4)
result1 := mathutil.FloorToFloat(3.14159, 1)
result2 := mathutil.FloorToFloat(3.14159, 2)
result3 := mathutil.FloorToFloat(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.1
// 3.14
// 5
// Output:
// 3.1
// 3.14
// 5
}
```
@@ -627,18 +627,18 @@ import (
)
func main() {
result1 := mathutil.FloorToString(3.14159, 1)
result2 := mathutil.FloorToString(3.14159, 2)
result3 := mathutil.FloorToString(5, 4)
result1 := mathutil.FloorToString(3.14159, 1)
result2 := mathutil.FloorToString(3.14159, 2)
result3 := mathutil.FloorToString(5, 4)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 3.1
// 3.14
// 5.0000
// Output:
// 3.1
// 3.14
// 5.0000
}
```
@@ -1114,17 +1114,17 @@ import (
func main() {
result1 := mathutil.Abs(-1)
result2 := mathutil.Abs(-0.1)
result3 := mathutil.Abs(float32(0.2))
result2 := mathutil.Abs(-0.1)
result3 := mathutil.Abs(float32(0.2))
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 1
// 0.1
// 0.2
// Output:
// 1
// 0.1
// 0.2
}
```
@@ -1149,16 +1149,16 @@ import (
)
func main() {
result1 := mathutil.Div(9, 4)
result2 := mathutil.Div(1, 2)
result3 := mathutil.Div(0, 666)
result1 := mathutil.Div(9, 4)
result2 := mathutil.Div(1, 2)
result3 := mathutil.Div(0, 666)
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 2.25
// 0.5
// 0
fmt.Println(result1)
fmt.Println(result2)
fmt.Println(result3)
// Output:
// 2.25
// 0.5
// 0
}
```
+203
View File
@@ -27,6 +27,10 @@ import (
- [RetryFunc](#RetryFunc)
- [RetryDuration](#RetryDuration)
- [RetryTimes](#RetryTimes)
- [BackoffStrategy](#BackoffStrategy)
- [RetryWithCustomBackoff](#RetryWithCustomBackoff)
- [RetryWithLinearBackoff](#RetryWithLinearBackoff)
- [RetryWithExponentialWithJitterBackoff](#RetryWithExponentialWithJitterBackoff)
<div STYLE="page-break-after: always;"></div>
@@ -259,3 +263,202 @@ func main() {
// 3
}
```
### <span id="BackoffStrategy">BackoffStrategy</span>
<p>An interface that defines a method for calculating backoff intervals.</p>
<b>Signature:</b>
```go
// BackoffStrategy is an interface that defines a method for calculating backoff intervals.
type BackoffStrategy interface {
// CalculateInterval returns the time.Duration after which the next retry attempt should be made.
CalculateInterval() time.Duration
}
```
<b>Example:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
type ExampleCustomBackoffStrategy struct {
interval time.Duration
}
func (c *ExampleCustomBackoffStrategy) CalculateInterval() time.Duration {
return c.interval + 1
}
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry,Retry(increaseNumber, retry.RetryWithCustomBackoff(&ExampleCustomBackoffStrategy{interval: time.Microsecond * 50}))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithCustomBackoff">RetryWithCustomBackoff</span>
<p>Set abitary custom backoff strategy.</p>
<b>Signature:</b>
```go
func RetryWithCustomBackoff(backoffStrategy BackoffStrategy) Option
```
<b>Example:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
type ExampleCustomBackoffStrategy struct {
interval time.Duration
}
func (c *ExampleCustomBackoffStrategy) CalculateInterval() time.Duration {
return c.interval + 1
}
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry,Retry(increaseNumber, retry.RetryWithCustomBackoff(&ExampleCustomBackoffStrategy{interval: time.Microsecond * 50}))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithLinearBackoff">RetryWithLinearBackoff</span>
<p>Set linear strategy backoff.</p>
<b>Signature:</b>
```go
func RetryWithLinearBackoff(interval time.Duration) Option
```
<b>Example:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry.Retry(increaseNumber, retry.RetryWithLinearBackoff(time.Microsecond*50))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
### <span id="RetryWithExponentialWithJitterBackoff">RetryWithExponentialWithJitterBackoff</span>
<p>Set exponential strategy backoff.</p>
<b>Signature:</b>
```go
func RetryWithExponentialWithJitterBackoff(interval time.Duration, base uint64, maxJitter time.Duration) Option
```
<b>Example:</b>
```go
package main
import (
"fmt"
"errors"
"log"
"github.com/duke-git/lancet/v2/retry"
)
func main() {
number := 0
increaseNumber := func() error {
number++
if number == 3 {
return nil
}
return errors.New("error occurs")
}
err := retry.Retry(increaseNumber, retry.RetryWithExponentialWithJitterBackoff(time.Microsecond*50, 2, time.Microsecond*25))
if err != nil {
return
}
fmt.Println(number)
// Output:
// 3
}
```
+1 -1
View File
@@ -30,7 +30,7 @@ We are excited that you are interested in contributing to lancet. Before submitt
- Before submitting a PR, please execute the unit test command: `go test -v ./...` to ensure that all unit test tasks should pass.
- Make sure PRs are created to `v2` branch instead of `master` branch.
- Make sure PRs are created to `rc` branch instead of other branch.
- If your PR fixes a bug, please provide a description about the related bug.
+1 -1
View File
@@ -30,7 +30,7 @@ Lancet 的成长离不开大家的支持,如果你愿意为 Lancet 贡献代
- 提交 PR 前请执行单元测试命令:go test -v ./...,确保所有单元测试任务通过。
- 确保 PR 是提交到 `v2` 分支,而不是 `main` 分支。
- 确保 PR 是提交到 `rc` 分支,而不是其他分支。
- 如果是修复 bug,请在 PR 中给出描述信息。
+30 -22
View File
@@ -16,7 +16,6 @@ import (
"fmt"
"io"
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
@@ -869,12 +868,13 @@ func isCsvSupportedType(v interface{}) bool {
}
}
// ChunkRead 从文件的指定偏移读取块并返回块内所有行
func ChunkRead(f *os.File, offset int64, size int, bufPool *sync.Pool) []string {
// ChunkRead reads a block from the file at the specified offset and returns all lines within the block
// Play: todo
func ChunkRead(file *os.File, offset int64, size int, bufPool *sync.Pool) ([]string, error) {
buf := bufPool.Get().([]byte)[:size] // 从Pool获取缓冲区并调整大小
n, err := f.ReadAt(buf, offset) // 从指定偏移读取数据到缓冲区
n, err := file.ReadAt(buf, offset) // 从指定偏移读取数据到缓冲区
if err != nil && err != io.EOF {
log.Fatal(err)
return nil, err
}
buf = buf[:n] // 调整切片以匹配实际读取的字节数
@@ -893,58 +893,64 @@ func ChunkRead(f *os.File, offset int64, size int, bufPool *sync.Pool) []string
lines = append(lines, line)
}
bufPool.Put(buf) // 读取完成后,将缓冲区放回Pool
return lines
return lines, nil
}
// 并行读取文件并将每个块的行发送到指定通道
// ParallelChunkRead reads the file in parallel and send each chunk of lines to the specified channel.
// filePath 文件路径
// ChunkSizeMB 分块的大小(单位MB,设置为0时使用默认100MB),设置过大反而不利,视情调整
// MaxGoroutine 并发读取分块的数量,设置为0时使用CPU核心数
// chunkSizeMB 分块的大小(单位MB,设置为0时使用默认100MB),设置过大反而不利,视情调整
// maxGoroutine 并发读取分块的数量,设置为0时使用CPU核心数
// linesCh用于接收返回结果的通道。
func ParallelChunkRead(filePath string, linesCh chan<- []string, ChunkSizeMB, MaxGoroutine int) {
if ChunkSizeMB == 0 {
ChunkSizeMB = 100
// Play: todo
func ParallelChunkRead(filePath string, linesCh chan<- []string, chunkSizeMB, maxGoroutine int) error {
if chunkSizeMB == 0 {
chunkSizeMB = 100
}
ChunkSize := ChunkSizeMB * 1024 * 1024
chunkSize := chunkSizeMB * 1024 * 1024
// 内存复用
bufPool := sync.Pool{
New: func() interface{} {
return make([]byte, 0, ChunkSize)
return make([]byte, 0, chunkSize)
},
}
if MaxGoroutine == 0 {
MaxGoroutine = runtime.NumCPU() // 设置为0时使用CPU核心数
if maxGoroutine == 0 {
maxGoroutine = runtime.NumCPU() // 设置为0时使用CPU核心数
}
f, err := os.Open(filePath)
if err != nil {
log.Fatalf("failed to open file: %v", err)
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
log.Fatalf("failed to get file info: %v", err)
return err
}
wg := sync.WaitGroup{}
chunkOffsetCh := make(chan int64, MaxGoroutine)
chunkOffsetCh := make(chan int64, maxGoroutine)
// 分配工作
go func() {
for i := int64(0); i < info.Size(); i += int64(ChunkSize) {
for i := int64(0); i < info.Size(); i += int64(chunkSize) {
chunkOffsetCh <- i
}
close(chunkOffsetCh)
}()
// 启动工作协程
for i := 0; i < MaxGoroutine; i++ {
for i := 0; i < maxGoroutine; i++ {
wg.Add(1)
go func() {
for chunkOffset := range chunkOffsetCh {
linesCh <- ChunkRead(f, chunkOffset, ChunkSize, &bufPool)
chunk, err := ChunkRead(f, chunkOffset, chunkSize, &bufPool)
if err == nil {
linesCh <- chunk
}
}
wg.Done()
}()
@@ -953,4 +959,6 @@ func ParallelChunkRead(filePath string, linesCh chan<- []string, ChunkSizeMB, Ma
// 等待所有解析完成后关闭行通道
wg.Wait()
close(linesCh)
return nil
}
+63
View File
@@ -5,6 +5,8 @@ import (
"io"
"log"
"os"
"runtime"
"sync"
)
func ExampleIsExist() {
@@ -421,8 +423,69 @@ func ExampleReadFile() {
if err != nil {
return
}
fmt.Println(string(dat))
// Output:
// User-agent: *
// Disallow: /deny
}
func ExampleChunkRead() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100
filePath := "./testdata/test1.csv"
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, 0, defaultChunkSizeMB*mb)
},
}
lines, err := ChunkRead(f, 0, 100, &bufPool)
if err != nil {
return
}
fmt.Println(lines[0])
fmt.Println(lines[1])
// Output:
// Lili,22,female
// Jim,21,male
}
func ExampleParallelChunkRead() {
const mb = 1024 * 1024
const defaultChunkSizeMB = 100 // 默认值
numParsers := runtime.NumCPU()
linesCh := make(chan []string, numParsers)
filePath := "./testdata/test1.csv"
go ParallelChunkRead(filePath, linesCh, defaultChunkSizeMB, numParsers)
var totalLines int
for lines := range linesCh {
totalLines += len(lines)
for _, line := range lines {
fmt.Println(line)
}
}
fmt.Println(totalLines)
// Output:
// Lili,22,female
// Jim,21,male
// 2
}
+53
View File
@@ -4,7 +4,9 @@ import (
"io"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"github.com/duke-git/lancet/v2/internal"
@@ -566,3 +568,54 @@ func TestCopyDir(t *testing.T) {
os.RemoveAll(dest)
}
func TestParallelChunkRead(t *testing.T) {
assert := internal.NewAssert(t, "TestParallelChunkRead")
const mb = 1024 * 1024
const defaultChunkSizeMB = 100 // 默认值
numParsers := runtime.NumCPU()
linesCh := make(chan []string, numParsers)
filePath := "./testdata/test1.csv" // 替换为你的文件路径
go ParallelChunkRead(filePath, linesCh, defaultChunkSizeMB, numParsers)
var totalLines int
for lines := range linesCh {
totalLines += len(lines)
assert.Equal("Lili,22,female", lines[0])
assert.Equal("Jim,21,male", lines[1])
}
assert.Equal(2, totalLines)
}
func TestChunkRead(t *testing.T) {
assert := internal.NewAssert(t, "TestChunkRead")
const mb = 1024 * 1024
const defaultChunkSizeMB = 100 // 默认值
filePath := "./testdata/test1.csv" // 替换为你的文件路径
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, 0, defaultChunkSizeMB*mb)
},
}
lines, err := ChunkRead(f, 0, 100, &bufPool)
assert.Equal("Lili,22,female", lines[0])
assert.Equal("Jim,21,male", lines[1])
}
-3
View File
@@ -1,5 +1,2 @@
Lili,22,female
Jim,21,male
1 Lili 22 female
2 Jim 21 male
+3 -3
View File
@@ -45,7 +45,7 @@ func RetryTimes(n uint) Option {
}
// RetryWithCustomBackoff set abitary custom backoff strategy
// todo: Add playground link
// Play: todo
func RetryWithCustomBackoff(backoffStrategy BackoffStrategy) Option {
if backoffStrategy == nil {
panic("programming error: backoffStrategy must be not nil")
@@ -57,7 +57,7 @@ func RetryWithCustomBackoff(backoffStrategy BackoffStrategy) Option {
}
// RetryWithLinearBackoff set linear strategy backoff
// todo: Add playground link
// Play: todo
func RetryWithLinearBackoff(interval time.Duration) Option {
if interval <= 0 {
panic("programming error: retry interval should not be lower or equal to 0")
@@ -71,7 +71,7 @@ func RetryWithLinearBackoff(interval time.Duration) Option {
}
// RetryWithExponentialWithJitterBackoff set exponential strategy backoff
// todo: Add playground link
// Play: todo
func RetryWithExponentialWithJitterBackoff(interval time.Duration, base uint64, maxJitter time.Duration) Option {
if interval <= 0 {
panic("programming error: retry interval should not be lower or equal to 0")
+1
View File
@@ -1173,6 +1173,7 @@ func AppendIfAbsent[T comparable](slice []T, item T) []T {
// SetToDefaultIf sets elements to their default value if they match the given predicate.
// It retains the positions of the elements in the slice.
// It returns slice of T and the count of modified slice items
// Play: todo
func SetToDefaultIf[T any](slice []T, predicate func(T) bool) ([]T, int) {
var count int
for i := 0; i < len(slice); i++ {