Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ctl: replace doRequest with HTTP client to get health status #8212

Merged
merged 18 commits into from
May 27, 2024
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions client/http/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const (
membersPrefix = "/pd/api/v1/members"
leaderPrefix = "/pd/api/v1/leader"
transferLeader = "/pd/api/v1/leader/transfer"
health = "/pd/api/v1/health"
// Config
Config = "/pd/api/v1/config"
ClusterVersion = "/pd/api/v1/config/cluster-version"
Expand Down
15 changes: 15 additions & 0 deletions client/http/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
GetStores(context.Context) (*StoresInfo, error)
GetStore(context.Context, uint64) (*StoreInfo, error)
SetStoreLabels(context.Context, int64, map[string]string) error
GetHealthStatus(context.Context) ([]Health, error)
/* Config-related interfaces */
GetConfig(context.Context) (map[string]any, error)
SetConfig(context.Context, map[string]any, ...float64) error
Expand Down Expand Up @@ -337,6 +338,20 @@
WithBody(jsonInput))
}

// GetHealthStatus gets the health status of the cluster.
func (c *client) GetHealthStatus(ctx context.Context) ([]Health, error) {
okJiang marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW, could you please add tests for this new method in /tests/integrations/client/http_client_test.go?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

var healths []Health
err := c.request(ctx, newRequestInfo().
WithName(getHealthStatusName).
WithURI(health).
WithMethod(http.MethodGet).
WithResp(&healths))
if err != nil {
return nil, err

Check warning on line 350 in client/http/interface.go

View check run for this annotation

Codecov / codecov/patch

client/http/interface.go#L350

Added line #L350 was not covered by tests
}
return healths, nil
}

// GetConfig gets the configurations.
func (c *client) GetConfig(ctx context.Context) (map[string]any, error) {
var config map[string]any
Expand Down
1 change: 1 addition & 0 deletions client/http/request_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const (
getStoresName = "GetStores"
getStoreName = "GetStore"
setStoreLabelsName = "SetStoreLabels"
getHealthStatusName = "GetHealthStatus"
getConfigName = "GetConfig"
setConfigName = "SetConfig"
getScheduleConfigName = "GetScheduleConfig"
Expand Down
9 changes: 9 additions & 0 deletions client/http/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -661,3 +661,12 @@ func stringToKeyspaceState(str string) (keyspacepb.KeyspaceState, error) {
return keyspacepb.KeyspaceState(0), fmt.Errorf("invalid KeyspaceState string: %s", str)
}
}

// Health reflects the cluster's health.
// NOTE: This type is moved from `server/api/health.go`, maybe move them to the same place later.
type Health struct {
Name string `json:"name"`
MemberID uint64 `json:"member_id"`
ClientUrls []string `json:"client_urls"`
Health bool `json:"health"`
}
1 change: 1 addition & 0 deletions pkg/dashboard/uiserver/embedded_assets_rewriter.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
var once sync.Once

// Assets returns the Assets FileSystem of the dashboard UI
// NOTE: if you see "undefined: assets" error, please run `make dashboard-ui` in the root directory of the repository.
func Assets(cfg *config.Config) http.FileSystem {
once.Do(func() {
resPath := distroutil.MustGetResPath()
Expand Down
19 changes: 19 additions & 0 deletions tests/integrations/client/http_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -811,3 +811,22 @@ func (suite *httpClientTestSuite) checkUpdateKeyspaceGCManagementType(mode mode,
re.True(ok)
re.Equal(expectGCManagementType, val)
}

func (suite *httpClientTestSuite) TestGetHealthStatus() {
suite.RunTestInTwoModes(suite.checkGetHealthStatus)
}

func (suite *httpClientTestSuite) checkGetHealthStatus(mode mode, client pd.Client) {
re := suite.Require()
env := suite.env[mode]

healths, err := client.GetHealthStatus(env.ctx)
re.NoError(err)
re.Len(healths, 2)
sort.Slice(healths, func(i, j int) bool {
return healths[i].Name < healths[j].Name
})
re.Equal("pd1", healths[0].Name)
re.Equal("pd2", healths[1].Name)
re.True(healths[0].Health && healths[1].Health)
}
17 changes: 6 additions & 11 deletions tools/pd-ctl/pdctl/command/health_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,25 @@
package command

import (
"net/http"

"github.com/spf13/cobra"
)

var (
healthPrefix = "pd/api/v1/health"
)

// NewHealthCommand return a health subcommand of rootCmd
func NewHealthCommand() *cobra.Command {
m := &cobra.Command{
Use: "health",
Short: "show all node's health information of the pd cluster",
Run: showHealthCommandFunc,
Use: "health",
Short: "show all node's health information of the PD cluster",
PersistentPreRunE: requirePDClient,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TestSendAndGetComponent ci failed :(

  • This is because the requirePDClient command executes the function shouldInitPDClient, which requires querying pd/api/v1/cluster.

  • The handler is replaced in TestSendAndGetComponent, so you need to add this router to the handler in TestSendAndGetComponent.

		mux.HandleFunc("/pd/api/v1/cluster", func(w http.ResponseWriter, r *http.Request) {
			cluster := &metapb.Cluster{
				Id: 0,
			}
			var rd render.Render
			rd.JSON(w, http.StatusOK, cluster)
		})

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can check code here

handler := func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
mux := http.NewServeMux()
mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) {
callerID := apiutil.GetCallerIDOnHTTP(r)
for k := range r.Header {
log.Info("header", zap.String("key", k))
}
log.Info("caller id", zap.String("caller-id", callerID))
re.Equal(pdControlCallerID, callerID)
fmt.Fprint(w, callerID)
})
info := apiutil.APIServiceGroup{
IsCore: true,
}
return mux, info, nil
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your reminder and suggestion. I will fix it tomorrow.

Run: showHealthCommandFunc,
}
return m
}

func showHealthCommandFunc(cmd *cobra.Command, _ []string) {
r, err := doRequest(cmd, healthPrefix, http.MethodGet, http.Header{})
health, err := PDCli.GetHealthStatus(cmd.Context())
if err != nil {
cmd.Println(err)
return
}
cmd.Println(r)
jsonPrint(cmd, health)
}
33 changes: 29 additions & 4 deletions tools/pd-ctl/tests/global_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,51 @@ package tests

import (
"context"
"fmt"
"encoding/json"
"net/http"
"testing"

"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/stretchr/testify/require"
"github.com/tikv/pd/pkg/utils/apiutil"
"github.com/tikv/pd/pkg/utils/assertutil"
"github.com/tikv/pd/pkg/utils/testutil"
"github.com/tikv/pd/server"
"github.com/tikv/pd/server/api"
cmd "github.com/tikv/pd/tools/pd-ctl/pdctl"
"go.uber.org/zap"
)

const pdControlCallerID = "pd-ctl"
const pdControlCallerID = "pd-http-client"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there need to be pd-ctl, maybe You can reuse the variables in the global.go file directly

pdControlCallerID = "pd-ctl"

FYI

func initNewPDClient(cmd *cobra.Command, opts ...pd.ClientOption) error {
if should, err := shouldInitPDClient(cmd); !should || err != nil {
return err
}
if PDCli != nil {
PDCli.Close()
}
PDCli = pd.NewClient(pdControlCallerID, getEndpoints(cmd), opts...)
return nil
}

Copy link
Member Author

@okJiang okJiang May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a little strange here. The unit test requires it to be pd-http-client. I'll take a look for it

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

c := &client{inner: newClientInner(ctx, cancel, source), callerID: defaultCallerID}

The HTTP client created in the pd-ctl is assigned by defaultCallerID. Do we need to change it to source. source equals to "pd-ctl" string.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is a bug :( Have been fixed in this pr. #8214

Copy link
Member Author

@okJiang okJiang May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you don't need open a new pr to fix it. We even need to modify the conflict for it. If it is bug, I can fix it in this pr easily. How was my propose?

Create an issue, link this pr, and then I can fix it.

Copy link
Member

@HuSharp HuSharp May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is a different issue, maybe with a new pr will be a little clearer 0 0

Because some background information is needed

After replacing HTTP with SDK #7490, we need to add the calledID explicitly to keep consistent with #4491
And source ID can be consistent with caller ID, #7565

Copy link
Member Author

@okJiang okJiang May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is ok for me. Let's wait for #8214 merged


func TestSendAndGetComponent(t *testing.T) {
re := require.New(t)
handler := func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
mux := http.NewServeMux()
mux.HandleFunc("/pd/api/v1/cluster", func(w http.ResponseWriter, _ *http.Request) {
cluster := &metapb.Cluster{
Id: 0,
}
clusterBytes, err := json.Marshal(cluster)
re.NoError(err)
w.Write(clusterBytes)
})
mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) {
callerID := apiutil.GetCallerIDOnHTTP(r)
for k := range r.Header {
log.Info("header", zap.String("key", k))
}
log.Info("caller id", zap.String("caller-id", callerID))
re.Equal(pdControlCallerID, callerID)
fmt.Fprint(w, callerID)
healths := []api.Health{
{
Name: "test",
},
}
healthsBytes, err := json.Marshal(healths)
re.NoError(err)
w.Write(healthsBytes)
Copy link
Member

@HuSharp HuSharp May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like this test wants to focus on calledID, so I think it's no need to change it.

FYI https://github.com/tikv/pd/pull/4491/files#diff-05102750a57e22445f5760fb1545227c9130a3a093f3b879e1f6e48ce27ea890

change cluster API because it is a required path

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

L55 has tested it. If we keep returning calledID, json.UnMarshal will be failed.

})
info := apiutil.APIServiceGroup{
IsCore: true,
Expand All @@ -67,5 +84,13 @@ func TestSendAndGetComponent(t *testing.T) {
args := []string{"-u", pdAddr, "health"}
output, err := ExecuteCommand(cmd, args...)
re.NoError(err)
re.Equal(fmt.Sprintf("%s\n", pdControlCallerID), string(output))
re.Equal(`[
{
"name": "test",
"member_id": 0,
"client_urls": null,
"health": false
}
]
`, string(output))
}