19. Distributed processing part 2: Data sharing between servers

iFun Engine provides a component called CrossServerStorage to easily share JSON data between servers through Redis.

You can use this component for the following:

  1. Check whether the client is authorized when moving between servers by issuing an authentication key

  2. Share number of players, transaction volume, etc. for each game region or server

  3. Share other game-related data

This chapter will provide examples of checking whether the client is authorized when moving between servers by issuing an authentication key.

Note

Not all interfaces of this component are explained in this example. For more, please refer to /usr/include/funapi/service/cross_server_storage.h or CrossServerStorage API documentation

.

19.1. Example - Sharing client authentication data when moving between servers

Assuming the following scenarios:

  1. Client login to game server

  2. Battle participation request

  3. The game server uses CrossServerStorage to issue a key and save user data needed for battle, then sends the key to the client

  4. The client contacts the battle server and sends the key it received from the game server

  5. The battle server receives the user information needed for battle through the key and handles the battle

  6. (Optional) Key removed from battle server

In other words, the game server issues an authentication key called key and the battle server checks whether the client is authenticated, then allows it to battle.

Note

Only the server-side code is suggested in this example, which doesn’t deal with client handling.

Important

Only synchronous functions are handled to simplify the explanation. It is better to use asynchronous functions on the actual server for better performance.

Important

The CrossServerStorage::Write() and CrossServerStorage::WriteSync() functions not dealt with in the example do not guarantee processing order when different servers write with the same key. Be aware that some data may be overwritten for that reason.

19.1.1. Setting up a project

19.1.1.1. Registering game and battle servers through flavors

Add APP_FLAVORS to the CMakeLists.txt file.

set(APP_FLAVORS game pvp)

When you build as we have done so far, src/MANIFEST.game.json and src/MANIFEST.pvp.json files are created.

19.1.1.2. Modifying MANIFEST.json

Modify the MANIFEST files as follows.

src/MANIFEST.game.json:

...

"SessionService": {
  "tcp_json_port": 8012,
  "udp_json_port": 0,
  "http_json_port": 0,
  "tcp_protobuf_port": 0,
  "udp_protobuf_port": 0,
  "http_protobuf_port": 0,
  ...
},
"Object": {
  "cache_expiration_in_ms": 3000,
  "copy_cache_expiration_in_ms": 700,
  "enable_database" : true,
  "db_mysql_server_address" : "tcp://10.10.10.10:3306",
  "db_mysql_id" : "funapi",
  "db_mysql_pw" : "funapi",
  "db_mysql_database" : "funapi",
  ...
},
"ApiService": {
  "api_service_port": 0,
  "api_service_event_tags_size": 1,
  "api_service_logging_level": 2
},
"Redis": {
  "enable_redis": true,
  "redis_mode": "redis",
  "redis_servers": {
    "cross_server_storage": {
      "address": "10.10.10.10:6379",
      "auth_pass": ""
    }
  },
  "redis_async_threads_size": 4
},
"RpcService": {
  "rpc_enabled": true,
  "rpc_threads_size": 4,
  "rpc_port": 8015,
  ...
},
"CrossServerStorage": {
  "enable_cross_server_storage": true,
  "redis_tag_for_cross_server_storage": "cross_server_storage"
},

...

src/MANIFEST.pvp.json:

...

"SessionService": {
  "tcp_json_port": 9012,
  "udp_json_port": 0,
  "http_json_port": 0,
  "tcp_protobuf_port": 0,
  "udp_protobuf_port": 0,
  "http_protobuf_port": 0,
  ...
},
"Object": {
  "cache_expiration_in_ms": 3000,
  "copy_cache_expiration_in_ms": 700,
  "enable_database" : true,
  "db_mysql_server_address" : "tcp://10.10.10.10:3306",
  "db_mysql_id" : "funapi",
  "db_mysql_pw" : "funapi",
  "db_mysql_database" : "funapi",
  ...
},
"ApiService": {
  "api_service_port": 0,
  "api_service_event_tags_size": 1,
  "api_service_logging_level": 2
},
"Redis": {
  "enable_redis": true,
  "redis_mode": "redis",
  "redis_servers": {
    "cross_server_storage": {
      "address": "10.10.10.10:6379",
      "auth_pass": ""
    }
  },
  "redis_async_threads_size": 4
},
"RpcService": {
  "rpc_enabled": true,
  "rpc_threads_size": 4,
  "rpc_port": 9015,
  ...
},
"CrossServerStorage": {
  "enable_cross_server_storage": true,
  "redis_tag_for_cross_server_storage": "cross_server_storage"
},

...

19.1.1.3. Setting up handling for each flavor

Modify the files as follows to handle for each flavor.

src/event_handlers.cc

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
// PLEASE ADD YOUR EVENT HANDLER DECLARATIONS HERE.

#include "event_handlers.h"

#include <funapi.h>
#include <glog/logging.h>

#include "test_loggers.h"
#include "test_messages.pb.h"

DECLARE_string(app_flavor);


namespace test {

void SendErrorMessage(const Ptr<Session> &session, const string &msg_type,
                      const string &error_message) {
  Json response;
  response["error"] = true;
  response["error_message"] = error_message;
  session->SendMessage(msg_type, response);
}


void RegisterEventHandlers() {
  if (FLAGS_app_flavor == "game") {
  } else if (FLAGS_app_flavor == "pvp") {
  }
}

}  // namespace test

mono/server.cs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
namespace Test
{

  public static void SendErrorMessage(
      Session session, string msg_type, string error_message)
  {
    JObject response = new JObject();
    response ["error"] = true;
    response ["error_message"] = error_message;
    session.SendMessage(msg_type, response);
  }


  public static void Install(ArgumentMap arguments)
  {
    ...
    if (Flags.GetString ("app_flavor") == "game")
    {
    }
    else if (Flags.GetString ("app_flavor") == "pvp")
    {
    }
  }
}

19.1.2. Registering login message handlers in the game flavor

Assuming a cs_game_server_login message is sent from the client to the server, register a OnGameServerLogin() handler to process this.

src/event_handlers.cc

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
void OnGameServerLogin(const Ptr<Session> &session, const Json &message) {
  const string user_name = message["user_name"].GetString();

  Ptr<User> user = User::FetchByName(user_name);
  if (not user) {
    user = User::Create(user_name);
    if (not user) {
      SendErrorMessage(session, "sc_game_server_login", "Already exists.");
      return;
    }

    user->SetLevel(1);
  }

  LOG_ASSERT(user);

  if (not AccountManager::CheckAndSetLoggedIn(user_name, session)) {
    SendErrorMessage(session, "sc_game_server_login", "Already logged in");
    return;
  }

  Json response;
  response["error"] = false;
  response["level"] = user->GetLevel();
  session->SendMessage("sc_game_server_login", response);
}


void RegisterEventHandlers() {
  if (FLAGS_app_flavor == "game") {
    HandlerRegistry::Register("cs_game_server_login", OnGameServerLogin);
  } else if (FLAGS_app_flavor == "pvp") {
    // 대전 서버의 메시지 핸들러를 등록합니다.
  }
}

mono/server.cs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
public static void OnGameServerLogin(
    Session session , JObject message)
{
  string user_name = (string) message ["user_name"];

  User user = User.FetchByName (user_name);
  if (user == null)
  {
    user = User.Create (user_name);

    if (user == null)
    {
      SendErrorMessage (session, "sc_game_server_login", "Already exists");
      return;
    }

    user.SetLevel (1);
  }

  Log.Assert (user != null);

  if (!AccountManager.CheckAndSetLoggedIn (user_name, session))
  {
    SendErrorMessage(session, "sc_game_server_login", "Already logged in");
    return;
  }

  JObject response = new JObject();
  response["error"] = false;
  response["level"] = user.GetLevel();
  session.SendMessage("sc_game_server_login", response);
}

public static void Install(ArgumentMap arguments)
{
  ...
  if (Flags.GetString ("app_flavor") == "game")
  {
    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_game_server_login",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnGameServerLogin));
  }
  else if (Flags.GetString ("app_flavor") == "pvp")
  {
    // 대전 서버의 메시지 핸들러를 등록합니다.
  }
}

19.1.3. Registering PvP mode messages in game flavors

Assuming a cs_enter_pvp message is sent from the client to the server, register a

OnEnterPvp() message handler to process this.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
void OnEnterPvp(const Ptr<Session> &session, const Json &message) {
  const string user_name = AccountManager::FindLocalAccount(session);
  if (user_name.empty()) {
    session->Close();
    return;
  }

  Ptr<User> user = User::FetchByName(user_name);
  if (not user) {
    SendErrorMessage(session, "sc_enter_pvp", "No user data");
    return;
  }

  Json user_data;
  user_data["user_name"] = user_name;
  user_data["level"] = user->GetLevel();

  // 클라이언트에게 전달할 인증키를 생성하고 CrossServerStorage 에 저장합니다.
  CrossServerStorage::Key auth_key;
  const CrossServerStorage::Result result = CrossServerStorage::CreateSync(
      user_data,
      WallClock::FromSec(10),  // 만료시간을 10 초로 지정합니다.
      &auth_key);

  if (result != CrossServerStorage::kSuccess) {
    LOG(ERROR) << "CrossServerStorage::CreateSync() failed: "
               << "result=" << result;
    SendErrorMessage(session, "sc_enter_pvp", "Server error");
    return;
  }

  const string pvp_server_ip = "10.10.10.10";
  const int64_t pvp_server_port = 9012;

  Json response;
  response["error"] = false;
  response["pvp_server_ip"] = pvp_server_ip;
  response["pvp_server_port"] = pvp_server_port;
  response["auth_key"] = boost::lexical_cast<string>(auth_key);
  session->SendMessage("sc_enter_pvp", response);

  // 이제 클라이언트는 대전 서버에 접속하여 인증키를 전달할 것입니다.
  // 대전 서버는 전달받은 인증키로 인증받은 클라이언트인지 확인할 것입니다.
}


void RegisterEventHandlers() {
  if (FLAGS_app_flavor == "game") {
    HandlerRegistry::Register("cs_game_server_login", OnGameServerLogin);
    HandlerRegistry::Register("cs_enter_pvp", OnEnterPvp);
  } else if (FLAGS_app_flavor == "pvp") {
    // 대전 서버의 메시지 핸들러를 등록합니다.
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
public static void OnEnterPvp(Session session , JObject message)
{
  string user_name = AccountManager.FindLocalAccount (session);
  if (user_name == String.Empty) {
    session.Close ();
    return;
  }

  User user = User.FetchByName (user_name);
  if (user == null) {
    SendErrorMessage (
        session, "sc_enter_pvp", "No user data");
    return;
  }

  JObject user_data = new JObject ();
  user_data["user_name"] = user_name;
  user_data["level"] = user.GetLevel ();

  // CrossServerStorage 를 이용해서 저장합니다.
  // 이때 인증키를 발급받아 클라이언트에게 전달합니다.
  Guid auth_key;
  CrossServerStorage.Result result = CrossServerStorage.CreateSync(
      user_data,
      WallClock.FromSec(10),  // 만료시간을 10 초로 지정합니다.
      out auth_key);

  if (result != CrossServerStorage.Result.kSuccess) {
    Log.Error ("CrossServerStorage::CreateSync() failed: result= {0}",
               result);
    SendErrorMessage (session, "sc_enter_pvp", "Server error");
    return;
  }

  string pvp_server_ip = "127.0.0.1";
  short pvp_server_port = 9012;

  JObject response = new JObject ();
  response ["error"] = false;
  response ["pvp_server_ip"] = pvp_server_ip;
  response ["pvp_server_port"] = pvp_server_port;
  response ["auth_key"] = auth_key.ToString();
  session.SendMessage ("sc_enter_pvp", response);

  // 이제 클라이언트는 대전 서버에 접속하여 인증키를 전달할 것입니다.
  // 대전 서버는 전달받은 인증키로 인증받은 클라이언트인지 확인할 것입니다.
}

public static void Install(ArgumentMap arguments)
{
  ...

  if (Flags.GetString ("app_flavor") == "game")
  {
    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_game_server_login",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnGameServerLogin));

    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_enter_pvp",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnEnterPvp));
  }
  else if (Flags.GetString ("app_flavor") == "pvp")
  {
    // 대전 서버의 메시지 핸들러를 등록합니다.
  }
}

Tip

If using the distributed processing feature Sharing status/data between servers, rather than hard coding a battle server accessed by the OnEnterPvp() function, you can choose a server with low load from among several battle servers.

19.1.4. Registering PvP participation messages in PvP flavors

Assuming a cs_transfer_pvp_server message is sent from the client to the server, register a

OnTransferPvpServer() message handler to process this.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
void OnTransferPvpServer(const Ptr<Session> &session, const Json &message) {
  if (not message.HasAttribute("auth_key", Json::kString) ||
      message["auth_key"].GetString().empty()) {
    session->Close();
    return;
  }

  CrossServerStorage::Key auth_key = CrossServerStorage::Key();
  try {
    auth_key =
        boost::lexical_cast<CrossServerStorage::Key>(
            message["auth_key"].GetString());
  } catch (const boost::bad_lexical_cast &) {
    session->Close();
    LOG(ERROR) << "인증키 얻기 실패. auth_key=" << message["auth_key"].GetString();
    return;
  }

  Json user_data;
  CrossServerStorage::Result result =
      CrossServerStorage::ReadSync(auth_key, &user_data);

  if (result != CrossServerStorage::kSuccess) {
    SendErrorMessage(session, "sc_transfer_pvp_server", "Failed to authenticate");
    return;
  }

  const string user_name = user_data["user_name"].GetString();
  const int64_t level = user_data["level"].GetInteger();

  LOG(INFO) << "user_name=" << user_name << ", level=" << level;

  // 이 예제에서는 인증키 만료시간이 10초이므로 명시적으로 삭제하지 않습니다.

  // 만약 명시적으로 인증키를 삭제해야 한다면 다음 함수로 삭제할 수 있습니다.
  // CrossServerStorage::DeleteSync(auth_key);

  // 인증키를 계속 사용할 수도 있습니다. (게임 서버가 대전 결과가 필요한 경우)
  // 아래 함수를 이용하면 인증키 만료 시간을 갱신할 수 있습니다.
  // CrossServerStorage::RefreshSync(auth_key, WallClock::FromSec(60));
}


void RegisterEventHandlers() {
  if (FLAGS_app_flavor == "game") {
    HandlerRegistry::Register("cs_game_server_login", OnGameServerLogin);
    HandlerRegistry::Register("cs_enter_pvp", OnEnterPvp);
  } else if (FLAGS_app_flavor == "pvp") {
    HandlerRegistry::Register("cs_transfer_pvp_server", OnTransferPvpServer);
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
public static void OnTransferPvpServer(
    Session session , JObject message)
{
  if (message ["auth_key"] == null)
  {
    session.Close ();
    return;
  }
  else if ( (string) message ["auth_key"] == String.Empty)
  {
    session.Close ();
  }

  Guid auth_key;

  try {
    auth_key = new Guid ((string) message ["auth_key"]);
  } catch (Exception exception) {
    session.Close();
    Log.Error ("인증키 얻기 실패. auth_key= {0}",
               (string)  message ["auth_key"]);
    return;
  }

  JObject user_data;
  CrossServerStorage.Result result =
      CrossServerStorage.ReadSync(auth_key, out user_data);

  if (result != CrossServerStorage.Result.kSuccess)
  {
    SendErrorMessage (session, "sc_transfer_pvp_server", "Failed to authenticate");
    return;
  }

  string user_name = (string) user_data ["user_name"];
  ulong level = (ulong) user_data ["level"];

  Log.Info ("user_name= {0}, level= {1}", user_name, level);

  // 이 예제에서는 인증키 만료시간이 10초이므로 명시적으로 삭제하지 않습니다.

  // 만약 명시적으로 인증키를 삭제해야 한다면 다음 함수로 삭제할 수 있습니다.
  // CrossServerStorage.DeleteSync(auth_key);

  // 인증키를 계속 사용할 수도 있습니다. (게임 서버가 대전 결과가 필요한 경우)
  // 아래 함수를 이용하면 인증키 만료 시간을 갱신할 수 있습니다.
  // CrossServerStorage.RefreshSync(auth_key, WallClock.FromSec(60));
}

public static void Install(ArgumentMap arguments)
{
  ...

  if (Flags.GetString ("app_flavor") == "game")
  {
    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_game_server_login",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnGameServerLogin));

    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_enter_pvp",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnEnterPvp));
  }
  else if (Flags.GetString ("app_flavor") == "pvp")
  {
    NetworkHandlerRegistry.RegisterMessageHandler (
        "cs_transfer_pvp_server",
        new NetworkHandlerRegistry.JsonMessageHandler (
            OnEnterPvp));
  }
}

19.2. CrossServerStorage parameters

  • enable_cross_server_storage: Whether or not to enable this feature. enable_redis from MANIFEST.json Configuration must be set to true.(type=bool, default=false)

  • redis_tag_for_cross_server_storage: Redis server tag used as storage to share data between servers. (See Distinguishing Redis Servers with Tags) (type=string, default=””)