From 60787202cba7eb1a5588030880c82bcb38f859b0 Mon Sep 17 00:00:00 2001 From: Cameron Reikes Date: Mon, 19 Jun 2023 05:17:06 -0700 Subject: [PATCH] Add binary serialization and serialization tests --- main.c | 362 ++++++++++++++++++++++++++++++++++++++++----------- makeprompt.h | 4 +- ser.h | 60 +++++++++ tuning.h | 5 +- 4 files changed, 347 insertions(+), 84 deletions(-) create mode 100644 ser.h diff --git a/main.c b/main.c index 603ec31..0cb10dd 100644 --- a/main.c +++ b/main.c @@ -120,6 +120,7 @@ void web_arena_set_auto_align(WebArena *arena, size_t align) #include "md.c" #pragma warning(pop) +#include "ser.h" #include @@ -559,7 +560,7 @@ void into_chunk(TextChunk *t, MD_String8 s) memcpy(t->text, s.str, s.size); t->text_length = (int)s.size; } -TextChunk *allocate_text_chunk() +TextChunk *allocate_text_chunk(MD_Arena *arena) { TextChunk *to_return = 0; if(text_chunk_free_list) @@ -569,7 +570,7 @@ TextChunk *allocate_text_chunk() } else { - to_return = MD_PushArray(persistent_arena, TextChunk, 1); + to_return = MD_PushArray(arena, TextChunk, 1); } *to_return = (TextChunk){0}; return to_return; @@ -590,7 +591,7 @@ int text_chunk_list_count(TextChunk *first) } void append_to_errors(Entity *from, MD_String8 s) { - TextChunk *error_chunk = allocate_text_chunk(); + TextChunk *error_chunk = allocate_text_chunk(persistent_arena); into_chunk(error_chunk, s); while(text_chunk_list_count(from->errorlist_first) > REMEMBERED_ERRORS) { @@ -1143,80 +1144,6 @@ MD_String8 is_action_valid(MD_Arena *arena, Entity *from, Action a) return error_message; } -#ifdef DEVTOOLS -void do_metadesk_tests() -{ - Log("Testing metadesk library...\n"); - MD_Arena *arena = MD_ArenaAlloc(); - MD_String8 s = MD_S8Lit("This is a testing|string"); - - MD_String8List split_up = MD_S8Split(arena, s, 1, &MD_S8Lit("|")); - - assert(split_up.node_count == 2); - assert(MD_S8Match(split_up.first->string, MD_S8Lit("This is a testing"), 0)); - assert(MD_S8Match(split_up.last->string, MD_S8Lit("string"), 0)); - - MD_ArenaRelease(arena); - - Log("Testing passed!\n"); -} -void do_parsing_tests() -{ - Log("Testing chatgpt parsing...\n"); - - MD_ArenaTemp scratch = MD_GetScratch(0, 0); - - Entity e = {0}; - e.npc_kind = NPC_TheBlacksmith; - e.exists = true; - Action a = {0}; - MD_String8 error; - MD_String8 speech; - - speech = MD_S8Lit("Better have a good reason for bothering me."); - MD_String8 thoughts = MD_S8Lit("Man I'm tired today Whatever."); - MD_String8 to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Meld\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); - error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); - assert(error.size == 0); - assert(a.kind == ACT_none); - assert(MD_S8Match(speech, MD_S8(a.speech, a.speech_length), 0)); - assert(MD_S8Match(thoughts, MD_S8(a.internal_monologue, a.internal_monologue_length), 0)); - - error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Chalice) \"Here you go\""), &a); - assert(error.size > 0); - error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Chalice) \""), &a); - assert(error.size > 0); - error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Cha \""), &a); - assert(error.size > 0); - - BUFF_APPEND(&e.held_items, ITEM_Chalice); - - error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(Chalice \""), &a); - assert(error.size > 0); - to_parse = MD_S8Lit("{action: give_item, action_arg: \"The Chalice of Gold\", speech: \"Here you go\", thoughts: \"Man I'm gonna miss that chalice\", who_i_am: \"Meld\", talking_to: nobody}"); - error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); - assert(error.size == 0); - assert(a.kind == ACT_give_item); - assert(a.argument.item_to_give == ITEM_Chalice); - - e.npc_kind = NPC_Door; - speech = MD_S8Lit("SAY THE WORDS"); - to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Ancient Door\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); - error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); - assert(error.size == 0); - error = is_action_valid(scratch.arena, &e, a); - assert(error.size == 0); - - speech = MD_S8Lit("THE WORD IS FOLLY"); - to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Ancient Door\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); - error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); - assert(error.size == 0); - error = is_action_valid(scratch.arena, &e, a); - assert(error.size > 0); - - MD_ReleaseScratch(scratch); -} -#endif // from must not be null @@ -1640,8 +1567,6 @@ void reset_level() { assert(ARRLEN(to_load->initial_entities) == ARRLEN(gs.entities)); memcpy(gs.entities, to_load->initial_entities, sizeof(Entity) * MAX_ENTITIES); - gs.version = CURRENT_VERSION; - for (Entity *it = gs.entities; it < gs.entities + ARRLEN(gs.entities); it++) { if(it->exists && it->generation == 0) @@ -1807,6 +1732,158 @@ void reset_level() } } +enum +{ + V0, + + VMax, +} Version; + +SER_MAKE_FOR_TYPE(uint64_t); +SER_MAKE_FOR_TYPE(bool); +SER_MAKE_FOR_TYPE(double); +SER_MAKE_FOR_TYPE(float); +SER_MAKE_FOR_TYPE(ItemKind); +SER_MAKE_FOR_TYPE(PropKind); +SER_MAKE_FOR_TYPE(NpcKind); +SER_MAKE_FOR_TYPE(CharacterState); +SER_MAKE_FOR_TYPE(Memory); +SER_MAKE_FOR_TYPE(Vec2); +SER_MAKE_FOR_TYPE(AnimKind); +SER_MAKE_FOR_TYPE(EntityRef); +SER_MAKE_FOR_TYPE(NPCPlayerStanding); + +#define SER_BUFF(ser, BuffElemType, buff_ptr) {ser_int(ser, &((buff_ptr)->cur_index));\ + if((buff_ptr)->cur_index > ARRLEN((buff_ptr)->data))\ + {\ + ser->cur_error = (SerError){.failed = true, .why = MD_S8Fmt(ser->error_arena, "Current index %d is more than the buffer %s's maximum, %d", (buff_ptr)->cur_index, #buff_ptr, ARRLEN((buff_ptr)->data))};\ + }\ + BUFF_ITER(BuffElemType, buff_ptr)\ + {\ + ser_##BuffElemType(ser, it);\ + }\ +} + +void ser_TextChunk(SerState *ser, TextChunk *t) +{ + ser_int(ser, &t->text_length); + if(t->text_length >= ARRLEN(t->text)) + { + ser->cur_error = (SerError){.failed = true, .why = MD_S8Fmt(ser->error_arena, "In text chunk, length %d is too big to fit into %d", t->text_length, ARRLEN(t->text))}; + } + ser_bytes(ser, (MD_u8*)t->text, t->text_length); +} + +void ser_entity(SerState *ser, Entity *e) +{ + ser_bool(ser, &e->destroy); + ser_int(ser, &e->generation); + + ser_Vec2(ser, &e->pos); + ser_Vec2(ser, &e->vel); + ser_float(ser, &e->damage); + + SER_BUFF(ser, ItemKind, &e->held_items); + + ser_bool(ser, &e->is_prop); + ser_PropKind(ser, &e->prop_kind); + + ser_bool(ser, &e->is_item); + ser_bool(ser, &e->held_by_player); + ser_ItemKind(ser, &e->item_kind); + + ser_bool(ser, &e->is_npc); + ser_bool(ser, &e->being_hovered); + ser_bool(ser, &e->perceptions_dirty); + + if(ser->serializing) + { + TextChunk *cur = e->errorlist_first; + bool more_errors = cur != 0; + ser_bool(ser, &more_errors); + while(more_errors) + { + ser_TextChunk(ser, cur); + cur = cur->next; + more_errors = cur != 0; + ser_bool(ser, &more_errors); + } + } + else + { + bool more_errors; + ser_bool(ser, &more_errors); + while(more_errors) + { + TextChunk *new_chunk = MD_PushArray(ser->arena, TextChunk, 1); + ser_TextChunk(ser, new_chunk); + MD_DblPushBack(e->errorlist_first, e->errorlist_last, new_chunk); + ser_bool(ser, &more_errors); + } + } + + ser_bool(ser, &e->opened); + ser_float(ser, &e->opened_amount); + ser_bool(ser, &e->gave_away_sword); + + SER_BUFF(ser, Memory, &e->memories); + + ser_bool(ser, &e->direction_of_spiral_pattern); + ser_float(ser, &e->dialog_panel_opacity); + ser_int(ser, &e->words_said); + ser_float(ser, &e->word_anim_in); + ser_NPCPlayerStanding(ser, &e->standing); + ser_NpcKind(ser, &e->npc_kind); + ser_int(ser, &e->gen_request_id); + ser_bool(ser, &e->walking); + ser_double(ser, &e->shotgun_timer); + ser_bool(ser, &e->moved); + ser_Vec2(ser, &e->target_goto); + // only for skeleton npc + ser_double(ser, &e->swing_timer); + + // character + ser_bool(ser, &e->is_character); + ser_bool(ser, &e->knighted); + ser_bool(ser, &e->in_conversation_mode); + ser_Vec2(ser, &e->to_throw_direction); + + SER_BUFF(ser, Vec2, &e->position_history); + ser_CharacterState(ser, &e->state); + ser_EntityRef(ser, &e->talking_to); + ser_bool(ser, &e->is_rolling); + ser_double(ser, &e->time_not_rolling); + + ser_AnimKind(ser, &e->cur_animation); + ser_float(ser, &e->anim_change_timer); +} + +void ser_GameState(SerState *ser, GameState *g) +{ + if(ser->serializing) ser->version = VMax - 1; + ser_int(ser, &ser->version); + if(ser->version >= VMax) + { + ser->cur_error = (SerError){.failed = true, .why = MD_S8Fmt(ser->error_arena, "Version %d is beyond the current version, %d", ser->version, VMax - 1)}; + } + + ser_uint64_t(ser, &g->tick); + ser_bool(ser, &g->won); + int num_entities = MAX_ENTITIES; + ser_int(ser, &num_entities); + + assert(num_entities <= MAX_ENTITIES); + for(int i = 0; i < num_entities; i++) + { + bool exists = gs.entities[i].exists; + ser_bool(ser, &exists); + if(exists) + { + ser_entity(ser, &gs.entities[i]); + } + } +} + #ifdef WEB EMSCRIPTEN_KEEPALIVE @@ -1971,6 +2048,132 @@ Vec2 img_size(sg_image img) return V2((float)info.width, (float)info.height); } +#ifdef DEVTOOLS +void do_metadesk_tests() +{ + Log("Testing metadesk library...\n"); + MD_Arena *arena = MD_ArenaAlloc(); + MD_String8 s = MD_S8Lit("This is a testing|string"); + + MD_String8List split_up = MD_S8Split(arena, s, 1, &MD_S8Lit("|")); + + assert(split_up.node_count == 2); + assert(MD_S8Match(split_up.first->string, MD_S8Lit("This is a testing"), 0)); + assert(MD_S8Match(split_up.last->string, MD_S8Lit("string"), 0)); + + MD_ArenaRelease(arena); + + Log("Testing passed!\n"); +} +void do_parsing_tests() +{ + Log("Testing chatgpt parsing...\n"); + + MD_ArenaTemp scratch = MD_GetScratch(0, 0); + + Entity e = {0}; + e.npc_kind = NPC_TheBlacksmith; + e.exists = true; + Action a = {0}; + MD_String8 error; + MD_String8 speech; + + speech = MD_S8Lit("Better have a good reason for bothering me."); + MD_String8 thoughts = MD_S8Lit("Man I'm tired today Whatever."); + MD_String8 to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Meld\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); + error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); + assert(error.size == 0); + assert(a.kind == ACT_none); + assert(MD_S8Match(speech, MD_S8(a.speech, a.speech_length), 0)); + assert(MD_S8Match(thoughts, MD_S8(a.internal_monologue, a.internal_monologue_length), 0)); + + error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Chalice) \"Here you go\""), &a); + assert(error.size > 0); + error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Chalice) \""), &a); + assert(error.size > 0); + error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(ITEM_Cha \""), &a); + assert(error.size > 0); + + BUFF_APPEND(&e.held_items, ITEM_Chalice); + + error = parse_chatgpt_response(scratch.arena, &e, MD_S8Lit("ACT_give_item(Chalice \""), &a); + assert(error.size > 0); + to_parse = MD_S8Lit("{action: give_item, action_arg: \"The Chalice of Gold\", speech: \"Here you go\", thoughts: \"Man I'm gonna miss that chalice\", who_i_am: \"Meld\", talking_to: nobody}"); + error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); + assert(error.size == 0); + assert(a.kind == ACT_give_item); + assert(a.argument.item_to_give == ITEM_Chalice); + + e.npc_kind = NPC_Door; + speech = MD_S8Lit("SAY THE WORDS"); + to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Ancient Door\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); + error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); + assert(error.size == 0); + error = is_action_valid(scratch.arena, &e, a); + assert(error.size == 0); + + speech = MD_S8Lit("THE WORD IS FOLLY"); + to_parse = FmtWithLint(scratch.arena, "{action: none, speech: \"%.*s\", thoughts: \"%.*s\", who_i_am: \"Ancient Door\", talking_to: nobody}", MD_S8VArg(speech), MD_S8VArg(thoughts)); + error = parse_chatgpt_response(scratch.arena, &e, to_parse, &a); + assert(error.size == 0); + error = is_action_valid(scratch.arena, &e, a); + assert(error.size > 0); + + MD_ReleaseScratch(scratch); +} +void do_serialization_tests() +{ + Log("Testing serialization...\n"); + + MD_ArenaTemp scratch = MD_GetScratch(0, 0); + + reset_level(); + player->pos = V2(50.0f, 0.0); + + MD_u8 *serialized_data = 0; + MD_u64 serialized_length = 0; + { + SerState ser = { + .serializing = true, + .error_arena = scratch.arena, + }; + ser_GameState(&ser, &gs); + + assert(!ser.cur_error.failed); + + ser.arena = scratch.arena; + ser.max = ser.cur; + ser.cur = 0; + ser.version = VMax - 1; + serialized_data = MD_ArenaPush(scratch.arena, ser.max); + ser.data = serialized_data; + + ser_GameState(&ser, &gs); + serialized_length = ser.cur; + player->pos.x = 0.0; + } + assert(serialized_length > 0); + assert(serialized_data != 0); + + reset_level(); + SerState ser = { + .serializing = false, + .data = serialized_data, + .max = serialized_length, + .arena = scratch.arena, + .error_arena = scratch.arena, + .version = VMax - 1, + }; + ser_GameState(&ser, &gs); + assert(player->pos.x == 50.0f); + assert(!ser.cur_error.failed); + + Log("Default save data size is %lld bytes\n", serialized_length); + + MD_ReleaseScratch(scratch); +} +#endif + void init(void) { #ifdef WEB @@ -1992,6 +2195,7 @@ void init(void) #ifdef DEVTOOLS do_metadesk_tests(); do_parsing_tests(); + do_serialization_tests(); #endif diff --git a/makeprompt.h b/makeprompt.h index d3adc80..a25fa48 100644 --- a/makeprompt.h +++ b/makeprompt.h @@ -87,6 +87,7 @@ MD_String8 escape_for_json(MD_Arena *arena, MD_String8 from) typedef struct { + // serialized as bytes. No pointers. ItemKind item_to_give; } ActionArgument; @@ -412,11 +413,8 @@ void fill_available_actions(Entity *it, AvailableActions *a) } } -#define MAX_ENTITIES 128 typedef struct GameState { - int version; // this field must be first to detect versions of old saves. Must bee consistent - uint64_t tick; bool won; Entity entities[MAX_ENTITIES]; diff --git a/ser.h b/ser.h new file mode 100644 index 0000000..b361378 --- /dev/null +++ b/ser.h @@ -0,0 +1,60 @@ +#pragma once + +#include "md.h" + +typedef struct +{ + MD_b8 failed; + MD_String8 why; +} SerError; + +typedef struct +{ + MD_u8 *data; // set to 0 to dry run and get maximum size. max doesn't matter in this case + MD_u64 cur; + MD_u64 max; + + MD_Arena *arena; // allocate everything new on this, so that if serialization fails allocations can be undone + + int version; + SerError cur_error; + MD_Arena *error_arena; // all error messages are allocated here + + MD_b8 serializing; +} SerState; + +void ser_bytes(SerState *ser, MD_u8 *bytes, MD_u64 bytes_size) +{ + if(!ser->cur_error.failed) + { + if(ser->data) + { + // maximum doesn't matter unless writing to data + if(ser->cur + bytes_size > ser->max) + { + ser->cur_error = (SerError){.failed = true, .why = MD_S8Lit("Too big bro")}; + } + else + { + if(ser->serializing) + { + memcpy(ser->data + ser->cur, bytes, bytes_size); + } + else + { + memcpy(bytes, ser->data + ser->cur, bytes_size); + } + } + + } + + ser->cur += bytes_size; + } +} + +#define SER_MAKE_FOR_TYPE(type) void ser_##type(SerState *ser, type *into) \ +{ \ + ser_bytes(ser, (MD_u8*)into, sizeof(*into)); \ +} + +SER_MAKE_FOR_TYPE(int); diff --git a/tuning.h b/tuning.h index e451feb..8ea732a 100644 --- a/tuning.h +++ b/tuning.h @@ -1,7 +1,5 @@ #pragma once -#define CURRENT_VERSION 12 // wehenver you change Entity increment this boz - #define LEVEL_TILES 150 // width and height of level tiles array #define LAYERS 3 #define TILE_SIZE 32 // in pixels @@ -27,6 +25,9 @@ #define IS_SERVER_SECURE 1 #endif +// this can never go down or else the forward compatibility of serialization breaks. +#define MAX_ENTITIES 128 + // REFACTORING:: also have to update in javascript!!!!!!!! #define MAX_SENTENCE_LENGTH 800 // LOOOK AT AGBOVE COMMENT GBEFORE CHANGING #define SENTENCE_CONST(txt) { .data = txt, .cur_index = sizeof(txt) }