Skip to content

Commit 2bf8de6

Browse files
jkaflikjkaflik
authored
Merge branch 'main' into ignore-mat-alias-cols
2 parents ffc756b + 4c8219e commit 2bf8de6

12 files changed

+250
-186
lines changed

‎.github/workflows/run-tests-cloud.yml

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: run-tests-cloud
2+
3+
permissions: write-all
4+
5+
on:
6+
push:
7+
branches:
8+
- main
9+
10+
jobs:
11+
cloud:
12+
runs-on: ubuntu-latest
13+
defaults:
14+
run:
15+
shell: bash
16+
strategy:
17+
fail-fast: true
18+
matrix:
19+
go:
20+
- "1.22"
21+
- "1.21"
22+
steps:
23+
- name: Check Out Code
24+
uses: actions/checkout@v3
25+
26+
- name: Resolve runner IP
27+
run: echo "TF_VAR_allowed_cidr=$(curl -s ifconfig.me)/32" >> $GITHUB_ENV
28+
29+
- name: Setup Terraform
30+
uses: hashicorp/setup-terraform@v2.0.3
31+
with:
32+
terraform_version: 1.3.4
33+
terraform_wrapper: false
34+
35+
- name: Terraform Init
36+
id: init
37+
working-directory: .github/cloud/
38+
run: terraform init
39+
40+
- name: Terraform Validate
41+
id: validate
42+
working-directory: .github/cloud/
43+
run: terraform validate -no-color
44+
45+
- name: Set service name for pull request
46+
if: github.event_name == 'pull_request'
47+
working-directory: .github/cloud/
48+
run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} PR${{ github.event.pull_request.number }} $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
49+
50+
- name: Set service name for push
51+
if: github.event_name == 'push'
52+
working-directory: .github/cloud/
53+
run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} $(git rev-parse --short HEAD) $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
54+
55+
- name: Terraform Apply
56+
working-directory: .github/cloud/
57+
id: apply
58+
run: terraform apply -no-color -auto-approve -input=false -lock=false
59+
env:
60+
TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
61+
TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
62+
TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
63+
TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
64+
TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}
65+
66+
- name: Set Host
67+
working-directory: .github/cloud/
68+
run: echo "CLICKHOUSE_HOST=$(terraform output -raw CLICKHOUSE_HOST)" >> $GITHUB_ENV
69+
70+
- name: Service ID
71+
working-directory: .github/cloud/
72+
run: terraform output -raw SERVICE_ID
73+
74+
- name: Install Go ${{ matrix.go }}
75+
uses: actions/setup-go@v2.1.5
76+
with:
77+
stable: false
78+
go-version: ${{ matrix.go }}
79+
80+
- name: Run tests
81+
env:
82+
CLICKHOUSE_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
83+
CLICKHOUSE_USE_DOCKER: false
84+
CLICKHOUSE_USE_SSL: true
85+
run: |
86+
CLICKHOUSE_DIAL_TIMEOUT=20 CLICKHOUSE_TEST_TIMEOUT=600s CLICKHOUSE_QUORUM_INSERT=2 make test
87+
88+
- name: Cleanup
89+
if: always()
90+
working-directory: .github/cloud/
91+
run: terraform destroy -no-color -auto-approve -input=false -lock=false
92+
env:
93+
TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
94+
TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
95+
TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
96+
TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
97+
TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}

‎.github/workflows/run-tests-head.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99

1010
jobs:
1111
test-ch-head:
12-
runs-on: [self-hosted, style-checker]
12+
runs-on: ubuntu-latest
1313
strategy:
1414
fail-fast: true
1515
matrix:

‎.github/workflows/run-tests.yml

+3-91
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@ on:
1212

1313
jobs:
1414
single-node:
15-
runs-on: [self-hosted, style-checker]
15+
runs-on: ubuntu-latest
1616
strategy:
1717
fail-fast: true
1818
matrix:
1919
go:
2020
- "1.22"
2121
- "1.21"
2222
clickhouse: # https://github.com/ClickHouse/ClickHouse/blob/master/SECURITY.md#scope-and-supported-versions
23-
- "latest" # 24.1
23+
- "latest" # 24.2
24+
- "24.1"
2425
- "23.12"
25-
- "23.11"
2626
- "23.8"
2727
- "23.3"
2828
steps:
@@ -37,91 +37,3 @@ jobs:
3737
- name: Run tests
3838
run: |
3939
CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test
40-
41-
integration-tests-cloud:
42-
runs-on: [self-hosted, style-checker]
43-
defaults:
44-
run:
45-
shell: bash
46-
strategy:
47-
fail-fast: true
48-
matrix:
49-
go:
50-
- "1.22"
51-
- "1.21"
52-
steps:
53-
- name: Check Out Code
54-
uses: actions/checkout@v3
55-
56-
- name: Resolve runner IP
57-
run: echo "TF_VAR_allowed_cidr=$(curl -s ifconfig.me)/32" >> $GITHUB_ENV
58-
59-
- name: Setup Terraform
60-
uses: hashicorp/setup-terraform@v2.0.3
61-
with:
62-
terraform_version: 1.3.4
63-
terraform_wrapper: false
64-
65-
- name: Terraform Init
66-
id: init
67-
working-directory: .github/cloud/
68-
run: terraform init
69-
70-
- name: Terraform Validate
71-
id: validate
72-
working-directory: .github/cloud/
73-
run: terraform validate -no-color
74-
75-
- name: Set service name for pull request
76-
if: github.event_name == 'pull_request'
77-
working-directory: .github/cloud/
78-
run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} PR${{ github.event.pull_request.number }} $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
79-
80-
- name: Set service name for push
81-
if: github.event_name == 'push'
82-
working-directory: .github/cloud/
83-
run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} $(git rev-parse --short HEAD) $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
84-
85-
- name: Terraform Apply
86-
working-directory: .github/cloud/
87-
id: apply
88-
run: terraform apply -no-color -auto-approve -input=false -lock=false
89-
env:
90-
TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
91-
TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
92-
TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
93-
TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
94-
TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}
95-
96-
- name: Set Host
97-
working-directory: .github/cloud/
98-
run: echo "CLICKHOUSE_HOST=$(terraform output -raw CLICKHOUSE_HOST)" >> $GITHUB_ENV
99-
100-
- name: Service ID
101-
working-directory: .github/cloud/
102-
run: terraform output -raw SERVICE_ID
103-
104-
- name: Install Go ${{ matrix.go }}
105-
uses: actions/setup-go@v2.1.5
106-
with:
107-
stable: false
108-
go-version: ${{ matrix.go }}
109-
110-
- name: Run tests
111-
env:
112-
CLICKHOUSE_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
113-
CLICKHOUSE_USE_DOCKER: false
114-
CLICKHOUSE_USE_SSL: true
115-
run: |
116-
CLICKHOUSE_DIAL_TIMEOUT=20 CLICKHOUSE_TEST_TIMEOUT=600s CLICKHOUSE_QUORUM_INSERT=2 make test
117-
118-
- name: Cleanup
119-
if: always()
120-
working-directory: .github/cloud/
121-
run: terraform destroy -no-color -auto-approve -input=false -lock=false
122-
env:
123-
TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
124-
TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
125-
TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
126-
TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
127-
TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}

‎clickhouse_rows.go

+13-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ func (r *rows) Next() (result bool) {
4646
}
4747
next:
4848
if r.row >= r.block.Rows() {
49+
if r.stream == nil {
50+
return false
51+
}
4952
select {
5053
case err := <-r.errors:
5154
if err != nil {
@@ -95,7 +98,16 @@ func (r *rows) Columns() []string {
9598
}
9699

97100
func (r *rows) Close() error {
98-
active := 2
101+
if r.errors == nil && r.stream == nil {
102+
return r.err
103+
}
104+
active := 0
105+
if r.errors != nil {
106+
active++
107+
}
108+
if r.stream != nil {
109+
active++
110+
}
99111
for {
100112
select {
101113
case _, ok := <-r.stream:

‎conn_batch.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,9 @@ func (c *connect) prepareBatch(ctx context.Context, query string, opts driver.Pr
4747
if len(colMatch) == 2 {
4848
columns = strings.Split(colMatch[1], ",")
4949
for i := range columns {
50-
columns[i] = strings.Trim(strings.TrimSpace(columns[i]), "`\"")
50+
// refers to https://clickhouse.com/docs/en/sql-reference/syntax#identifiers
51+
// we can use identifiers with double quotes or backticks, for example: "id", `id`, but not both, like `"id"`.
52+
columns[i] = strings.Trim(strings.Trim(strings.TrimSpace(columns[i]), "\""), "`")
5153
}
5254
}
5355
if !strings.HasSuffix(strings.TrimSpace(strings.ToUpper(query)), "VALUES") {

‎conn_http.go

+21-45
Original file line numberDiff line numberDiff line change
@@ -75,49 +75,32 @@ type HTTPReaderWriter struct {
7575
method CompressionMethod
7676
}
7777

78-
func (rw HTTPReaderWriter) read(res *http.Response) ([]byte, error) {
78+
// NewReader will return a reader that will decompress data if needed.
79+
func (rw *HTTPReaderWriter) NewReader(res *http.Response) (io.Reader, error) {
7980
enc := res.Header.Get("Content-Encoding")
8081
if !res.Uncompressed && rw.method.String() == enc {
8182
switch rw.method {
8283
case CompressionGZIP:
8384
reader := rw.reader.(*gzip.Reader)
84-
defer reader.Close()
8585
if err := reader.Reset(res.Body); err != nil {
8686
return nil, err
8787
}
88-
body, err := io.ReadAll(reader)
89-
if err != nil {
90-
return nil, err
91-
}
92-
return body, nil
88+
return reader, nil
9389
case CompressionDeflate:
94-
reader := rw.reader.(io.ReadCloser)
95-
defer reader.Close()
96-
if err := rw.reader.(flate.Resetter).Reset(res.Body, nil); err != nil {
97-
return nil, err
98-
}
99-
body, err := io.ReadAll(reader)
100-
if err != nil {
90+
reader := rw.reader
91+
if err := reader.(flate.Resetter).Reset(res.Body, nil); err != nil {
10192
return nil, err
10293
}
103-
return body, nil
94+
return reader, nil
10495
case CompressionBrotli:
10596
reader := rw.reader.(*brotli.Reader)
10697
if err := reader.Reset(res.Body); err != nil {
10798
return nil, err
10899
}
109-
body, err := io.ReadAll(reader)
110-
if err != nil {
111-
return nil, err
112-
}
113-
return body, nil
100+
return reader, nil
114101
}
115102
}
116-
body, err := io.ReadAll(res.Body)
117-
if err != nil {
118-
return nil, err
119-
}
120-
return body, nil
103+
return res.Body, nil
121104
}
122105

123106
func (rw *HTTPReaderWriter) reset(pw *io.PipeWriter) io.WriteCloser {
@@ -436,27 +419,21 @@ func (h *httpConnect) sendQuery(ctx context.Context, query string, options *Quer
436419

437420
func (h *httpConnect) readRawResponse(response *http.Response) (body []byte, err error) {
438421
rw := h.compressionPool.Get()
439-
defer response.Body.Close()
440422
defer h.compressionPool.Put(rw)
441-
if body, err = rw.read(response); err != nil {
423+
424+
reader, err := rw.NewReader(response)
425+
if err != nil {
442426
return nil, err
443427
}
444428
if h.compression == CompressionLZ4 || h.compression == CompressionZSTD {
445-
result := make([]byte, len(body))
446-
reader := chproto.NewReader(bytes.NewReader(body))
447-
reader.EnableCompression()
448-
defer reader.DisableCompression()
449-
for {
450-
b, err := reader.ReadByte()
451-
if err != nil {
452-
if errors.Is(err, io.EOF) {
453-
break
454-
}
455-
return nil, err
456-
}
457-
result = append(result, b)
458-
}
459-
return result, nil
429+
chReader := chproto.NewReader(reader)
430+
chReader.EnableCompression()
431+
reader = chReader
432+
}
433+
434+
body, err = io.ReadAll(reader)
435+
if err != nil {
436+
return nil, err
460437
}
461438
return body, nil
462439
}
@@ -549,14 +526,13 @@ func (h *httpConnect) executeRequest(req *http.Request) (*http.Response, error)
549526
if err != nil {
550527
return nil, err
551528
}
552-
if resp.StatusCode != http.StatusOK {
553529

530+
if resp.StatusCode != http.StatusOK {
531+
defer resp.Body.Close()
554532
msg, err := h.readRawResponse(resp)
555-
556533
if err != nil {
557534
return nil, fmt.Errorf("clickhouse [execute]:: %d code: failed to read the response: %w", resp.StatusCode, err)
558535
}
559-
560536
return nil, fmt.Errorf("clickhouse [execute]:: %d code: %s", resp.StatusCode, string(msg))
561537
}
562538
return resp, nil

0 commit comments

Comments
 (0)