Receive and decode Audio Frames

CMakeLists.txt

@@ -5,6 +5,8 @@ project(chiaki)
 
 option(CHIAKI_ENABLE_TESTS "Enable tests for Chiaki" ON)
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
 add_subdirectory(third-party)
 
 add_subdirectory(lib)

cmake/FindOpus.cmake

@@ -0,0 +1,16 @@
+#  Opus_FOUND
+#  Opus_INCLUDE_DIRS
+#  Opus_LIBRARIES
+
+find_path(Opus_INCLUDE_DIRS
+	NAMES opus/opus.h
+	PATH_SUFFIXES include
+)
+
+find_library(Opus_LIBRARIES NAMES opus)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Opus
+	DEFAULT_MSG
+	Opus_INCLUDE_DIRS Opus_LIBRARIES
+)

gui/main.c

@@ -4,9 +4,9 @@
 #include <stdio.h>
 #include <string.h>
 
-void audio_frame_cb(uint8_t *buf, size_t buf_size, void *user)
+void audio_frame_cb(int8_t *buf, size_t samples_count, void *user)
 {
-	printf("AUDIO FRAME CB %lu\n", buf_size);
+	printf("AUDIO FRAME CB %lu\n", samples_count);
 }
 
 int main(int argc, const char *argv[])

lib/CMakeLists.txt

@@ -43,6 +43,9 @@ include_directories("${NANOPB_SOURCE_DIR}")
 set_source_files_properties(${CHIAKI_LIB_PROTO_SOURCE_FILES} ${CHIAKI_LIB_PROTO_HEADER_FILES} PROPERTIES GENERATED TRUE)
 include_directories("${CHIAKI_LIB_PROTO_INCLUDE_DIR}")
 
+find_package(Opus REQUIRED)
+include_directories(${Opus_INCLUDE_DIRS})
+
 add_library(chiaki-lib ${HEADER_FILES} ${SOURCE_FILES} ${CHIAKI_LIB_PROTO_SOURCE_FILES} ${CHIAKI_LIB_PROTO_HEADER_FILES})
 add_dependencies(chiaki-lib chiaki-pb)
 set_target_properties(chiaki-lib PROPERTIES OUTPUT_NAME chiaki)
@@ -55,4 +58,6 @@ target_link_libraries(chiaki-lib Threads::Threads)
 find_package(OpenSSL REQUIRED)
 target_link_libraries(chiaki-lib OpenSSL::Crypto)
 
-target_link_libraries(chiaki-lib protobuf-nanopb-static)
+target_link_libraries(chiaki-lib protobuf-nanopb-static)
+
+target_link_libraries(chiaki-lib ${Opus_LIBRARIES})

lib/include/chiaki/audioreceiver.h

@@ -20,20 +20,49 @@
 
 #include "common.h"
 #include "log.h"
+#include "audio.h"
+#include "thread.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+
 typedef struct chiaki_audio_receiver_t
 {
 	struct chiaki_session_t *session;
 	ChiakiLog *log;
+	ChiakiMutex mutex;
+	struct OpusDecoder *opus_decoder;
+	ChiakiAudioHeader audio_header;
 } ChiakiAudioReceiver;
 
-CHIAKI_EXPORT void chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);
+CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);
 CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receiver);
+CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_receiver, ChiakiAudioHeader *audio_header);
+CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size);
 
+static inline ChiakiAudioReceiver *chiaki_audio_receiver_new(struct chiaki_session_t *session)
+{
+	ChiakiAudioReceiver *audio_receiver = CHIAKI_NEW(ChiakiAudioReceiver);
+	if(!audio_receiver)
+		return NULL;
+	ChiakiErrorCode err = chiaki_audio_receiver_init(audio_receiver, session);
+	if(err != CHIAKI_ERR_SUCCESS)
+	{
+		free(audio_receiver);
+		return NULL;
+	}
+	return audio_receiver;
+}
+
+static inline void chiaki_audio_receiver_free(ChiakiAudioReceiver *audio_receiver)
+{
+	if(!audio_receiver)
+		return;
+	chiaki_audio_receiver_fini(audio_receiver);
+	free(audio_receiver);
+}
 
 #ifdef __cplusplus
 }

lib/include/chiaki/nagare.h

@@ -23,7 +23,6 @@
 #include "log.h"
 #include "ecdh.h"
 #include "gkcrypt.h"
-#include "audio.h"
 
 #include <stdbool.h>
 
@@ -40,7 +39,6 @@ typedef struct chiaki_nagare_t
 	uint8_t *ecdh_secret;
 	ChiakiGKCrypt *gkcrypt_a;
 	ChiakiGKCrypt *gkcrypt_b;
-	ChiakiAudioHeader audio_header;
 } ChiakiNagare;
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_nagare_run(struct chiaki_session_t *session);

lib/include/chiaki/session.h

@@ -27,6 +27,7 @@
 #include "takion.h"
 #include "ecdh.h"
 #include "audio.h"
+#include "audioreceiver.h"
 
 #include <stdint.h>
 #include <netdb.h>
@@ -88,7 +89,7 @@ typedef struct chiaki_event_t
 } ChiakiEvent;
 
 typedef void (*ChiakiEventCallback)(ChiakiEvent *event, void *user);
-typedef void (*ChiakiAudioFrameCallback)(uint8_t *buf, size_t buf_size, void *user);
+typedef void (*ChiakiAudioFrameCallback)(int16_t *buf, size_t samples_count, void *user);
 
 
 
@@ -131,6 +132,7 @@ typedef struct chiaki_session_t
 	ChiakiLog log;
 
 	ChiakiNagare nagare;
+	ChiakiAudioReceiver *audio_receiver;
 } ChiakiSession;
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_session_init(ChiakiSession *session, ChiakiConnectInfo *connect_info);

lib/include/chiaki/takion.h

@@ -31,6 +31,7 @@ extern "C" {
 
 
 typedef void (*ChiakiTakionDataCallback)(uint8_t *buf, size_t buf_size, void *user);
+typedef void (*ChiakiTakionAVCallback)(uint8_t *buf, size_t buf_size, uint32_t key_pos, void *user);
 
 
 typedef struct chiaki_takion_connect_info_t
@@ -40,6 +41,8 @@ typedef struct chiaki_takion_connect_info_t
 	socklen_t sa_len;
 	ChiakiTakionDataCallback data_cb;
 	void *data_cb_user;
+	ChiakiTakionAVCallback av_cb;
+	void *av_cb_user;
 } ChiakiTakionConnectInfo;
 
 
@@ -48,6 +51,8 @@ typedef struct chiaki_takion_t
 	ChiakiLog *log;
 	ChiakiTakionDataCallback data_cb;
 	void *data_cb_user;
+	ChiakiTakionAVCallback av_cb;
+	void *av_cb_user;
 	int sock;
 	ChiakiThread thread;
 	int stop_pipe[2];

lib/src/audio.c

@@ -22,8 +22,8 @@
 
 void chiaki_audio_header_load(ChiakiAudioHeader *audio_header, const uint8_t *buf)
 {
-	audio_header->bits = buf[0];
-	audio_header->channels = buf[1];
+	audio_header->channels = buf[0];
+	audio_header->bits = buf[1];
 	audio_header->rate = ntohl(*((uint32_t *)(buf + 2)));
 	audio_header->frame_size = ntohl(*((uint32_t *)(buf + 6)));
 	audio_header->unknown = ntohl(*((uint32_t *)(buf + 0xa)));

lib/src/audioreceiver.c

@@ -18,16 +18,82 @@
 #include <chiaki/audioreceiver.h>
 #include <chiaki/session.h>
 
+#include <opus/opus.h>
 
-CHIAKI_EXPORT void chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, ChiakiSession *session)
+#include <string.h>
+
+
+CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, ChiakiSession *session)
 {
 	audio_receiver->session = session;
 	audio_receiver->log = &session->log;
+	audio_receiver->opus_decoder = NULL;
+	memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header));
 
+	ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex);
+	if(err != CHIAKI_ERR_SUCCESS)
+		return err;
+
+	return CHIAKI_ERR_SUCCESS;
 }
 
 
 CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receiver)
 {
+	opus_decoder_destroy(audio_receiver->opus_decoder);
+	chiaki_mutex_fini(&audio_receiver->mutex);
+}
 
+
+CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_receiver, ChiakiAudioHeader *audio_header)
+{
+	chiaki_mutex_lock(&audio_receiver->mutex);
+
+	CHIAKI_LOGI(audio_receiver->log, "Audio Header:\n");
+	CHIAKI_LOGI(audio_receiver->log, "  channels = %d\n", audio_header->channels);
+	CHIAKI_LOGI(audio_receiver->log, "  bits = %d\n", audio_header->bits);
+	CHIAKI_LOGI(audio_receiver->log, "  rate = %d\n", audio_header->rate);
+	CHIAKI_LOGI(audio_receiver->log, "  frame size = %d\n", audio_header->frame_size);
+	CHIAKI_LOGI(audio_receiver->log, "  unknown = %d\n", audio_header->unknown);
+	memcpy(&audio_receiver->audio_header, audio_header, sizeof(audio_receiver->audio_header));
+
+	opus_decoder_destroy(audio_receiver->opus_decoder);
+
+	int error;
+	audio_receiver->opus_decoder = opus_decoder_create(audio_header->rate, audio_header->channels, &error);
+
+	if(error != OPUS_OK)
+		CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to initialize opus decoder: %s\n", opus_strerror(error));
+	else
+		CHIAKI_LOGI(audio_receiver->log, "Audio Receiver initialized opus decoder with the settings above\n");
+
+	chiaki_mutex_unlock(&audio_receiver->mutex);
+}
+
+
+CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size)
+{
+	chiaki_mutex_lock(&audio_receiver->mutex);
+
+	if(!audio_receiver->opus_decoder)
+	{
+		CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized\n");
+		chiaki_mutex_unlock(&audio_receiver->mutex);
+		return;
+	}
+
+	// TODO: don't malloc
+	opus_int16 *pcm = malloc(audio_receiver->audio_header.frame_size * audio_receiver->audio_header.channels * sizeof(opus_int16));
+
+	int r = opus_decode(audio_receiver->opus_decoder, buf, (opus_int32)buf_size, pcm, audio_receiver->audio_header.frame_size, 0);
+	if(r < 1)
+		CHIAKI_LOGE(audio_receiver->log, "Decoding audio frame with opus failed: %s\n", opus_strerror(r));
+	else
+	{
+		audio_receiver->session->audio_frame_cb(pcm, (size_t)r, audio_receiver->session->audio_frame_cb_user);
+	}
+
+	free(pcm);
+
+	chiaki_mutex_unlock(&audio_receiver->mutex);
 }

lib/src/nagare.c

@@ -58,6 +58,7 @@ static ChiakiErrorCode nagare_send_disconnect(ChiakiNagare *nagare);
 static void nagare_takion_data_expect_bang(ChiakiNagare *nagare, uint8_t *buf, size_t buf_size);
 static void nagare_takion_data_expect_streaminfo(ChiakiNagare *nagare, uint8_t *buf, size_t buf_size);
 static ChiakiErrorCode nagare_send_streaminfo_ack(ChiakiNagare *nagare);
+static void nagare_takion_av(uint8_t *buf, size_t buf_size, uint32_t key_pos, void *user);
 
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_nagare_run(ChiakiSession *session)
@@ -87,6 +88,8 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_nagare_run(ChiakiSession *session)
 
 	takion_info.data_cb = nagare_takion_data;
 	takion_info.data_cb_user = nagare;
+	takion_info.av_cb = nagare_takion_av;
+	takion_info.av_cb_user = nagare;
 
 	err = chiaki_takion_connect(&nagare->takion, &takion_info);
 	free(takion_info.sa);
@@ -326,7 +329,9 @@ static void nagare_takion_data_expect_streaminfo(ChiakiNagare *nagare, uint8_t *
 		goto error;
 	}
 
-	chiaki_audio_header_load(&nagare->audio_header, audio_header);
+	ChiakiAudioHeader audio_header_s;
+	chiaki_audio_header_load(&audio_header_s, audio_header);
+	chiaki_audio_receiver_stream_info(nagare->session->audio_receiver, &audio_header_s);
 
 	// TODO: do some checks?
 
@@ -488,3 +493,17 @@ static ChiakiErrorCode nagare_send_disconnect(ChiakiNagare *nagare)
 }
 
 
+static void nagare_takion_av(uint8_t *buf, size_t buf_size, uint32_t key_pos, void *user)
+{
+	ChiakiNagare *nagare = user;
+
+	chiaki_gkcrypt_decrypt(nagare->gkcrypt_b, key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, buf, buf_size);
+
+	if(buf[0] == 0xf4 && buf_size >= 0x50)
+	{
+		chiaki_audio_receiver_frame_packet(nagare->session->audio_receiver, buf, 0x50);
+	}
+
+	//CHIAKI_LOGD(nagare->log, "Nagare AV %lu\n", buf_size);
+	//chiaki_log_hexdump(nagare->log, CHIAKI_LOG_DEBUG, buf, buf_size);
+}

lib/src/session.c

@@ -15,6 +15,7 @@
  * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+#include <chiaki/audioreceiver.h>
 #include <chiaki/senkusha.h>
 #include <chiaki/session.h>
 #include <chiaki/http.h>
@@ -184,15 +185,26 @@ static void *session_thread_func(void *arg)
 		goto quit_ctrl;
 	}
 
+	session->audio_receiver = chiaki_audio_receiver_new(session);
+	if(!session->audio_receiver)
+	{
+		CHIAKI_LOGE(&session->log, "Session failed to initialize Audio Receiver\n");
+		goto quit_ctrl;
+	}
+
 	err = chiaki_nagare_run(session);
 	if(err != CHIAKI_ERR_SUCCESS)
 	{
 		CHIAKI_LOGE(&session->log, "Nagare failed\n");
-		goto quit_ctrl;
+		goto quit_audio_receiver;
 	}
 
 	CHIAKI_LOGI(&session->log, "Nagare completed successfully\n");
 
+quit_audio_receiver:
+	chiaki_audio_receiver_free(session->audio_receiver);
+	session->audio_receiver = NULL;
+
 quit_ctrl:
 	chiaki_ctrl_join(&session->ctrl);
 	CHIAKI_LOGI(&session->log, "Ctrl stopped\n");

lib/src/takion.c

@@ -96,6 +96,7 @@ static ChiakiErrorCode takion_send_message_cookie(ChiakiTakion *takion, uint8_t
 static ChiakiErrorCode takion_recv(ChiakiTakion *takion, uint8_t *buf, size_t *buf_size, struct timeval *timeout);
 static ChiakiErrorCode takion_recv_message_init_ack(ChiakiTakion *takion, TakionMessagePayloadInitAck *payload);
 static ChiakiErrorCode takion_recv_message_cookie_ack(ChiakiTakion *takion);
+static void takion_handle_packet_av(ChiakiTakion *takion, uint8_t base_type, uint8_t *buf, size_t buf_size);
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_connect(ChiakiTakion *takion, ChiakiTakionConnectInfo *info)
 {
@@ -104,6 +105,8 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_connect(ChiakiTakion *takion, Chiaki
 	takion->log = info->log;
 	takion->data_cb = info->data_cb;
 	takion->data_cb_user = info->data_cb_user;
+	takion->av_cb = info->av_cb;
+	takion->av_cb_user = info->av_cb_user;
 	takion->something = TAKION_LOCAL_SOMETHING;
 
 	takion->tag_local = 0x4823; // "random" tag
@@ -256,8 +259,8 @@ CHIAKI_EXPORT void chiaki_takion_close(ChiakiTakion *takion)
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_send_raw(ChiakiTakion *takion, uint8_t *buf, size_t buf_size)
 {
-	CHIAKI_LOGD(takion->log, "Takion send:\n");
-	chiaki_log_hexdump(takion->log, CHIAKI_LOG_DEBUG, buf, buf_size);
+	//CHIAKI_LOGD(takion->log, "Takion send:\n");
+	//chiaki_log_hexdump(takion->log, CHIAKI_LOG_DEBUG, buf, buf_size);
 
 	ssize_t r = send(takion->sock, buf, buf_size, 0);
 	if(r < 0)
@@ -369,16 +372,15 @@ static ChiakiErrorCode takion_recv(ChiakiTakion *takion, uint8_t *buf, size_t *b
 static void takion_handle_packet(ChiakiTakion *takion, uint8_t *buf, size_t buf_size)
 {
 	assert(buf_size > 0);
-	switch(buf[0])
+	uint8_t base_type = buf[0];
+	switch(base_type)
 	{
 		case TAKION_PACKET_TYPE_MESSAGE:
 			takion_handle_packet_message(takion, buf+1, buf_size-1);
 			break;
 		case TAKION_PACKET_TYPE_2:
-			CHIAKI_LOGW(takion->log, "TODO: Handle Takion Packet type 2\n");
-			break;
 		case TAKION_PACKET_TYPE_3:
-			CHIAKI_LOGW(takion->log, "TODO: Handle Takion Packet type 3\n");
+			takion_handle_packet_av(takion, base_type, buf+1, buf_size-1);
 			break;
 		default:
 			CHIAKI_LOGW(takion->log, "Takion packet with unknown type %#x received\n", buf[0]);
@@ -394,8 +396,8 @@ static void takion_handle_packet_message(ChiakiTakion *takion, uint8_t *buf, siz
 	if(err != CHIAKI_ERR_SUCCESS)
 		return;
 
-	CHIAKI_LOGD(takion->log, "Takion received message with tag %#x, key pos %#x, type (%#x, %#x), payload size %#x, payload:\n", msg.tag, msg.key_pos, msg.type_a, msg.type_b, msg.payload_size);
-	chiaki_log_hexdump(takion->log, CHIAKI_LOG_DEBUG, msg.payload, msg.payload_size);
+	//CHIAKI_LOGD(takion->log, "Takion received message with tag %#x, key pos %#x, type (%#x, %#x), payload size %#x, payload:\n", msg.tag, msg.key_pos, msg.type_a, msg.type_b, msg.payload_size);
+	//chiaki_log_hexdump(takion->log, CHIAKI_LOG_DEBUG, msg.payload, msg.payload_size);
 
 	switch(msg.type_a)
 	{
@@ -637,4 +639,30 @@ static ChiakiErrorCode takion_recv_message_cookie_ack(ChiakiTakion *takion)
 	assert(msg.payload_size == 0);
 
 	return CHIAKI_ERR_SUCCESS;
+}
+
+
+#define AV_HEADER_SIZE 0x12
+
+static void takion_handle_packet_av(ChiakiTakion *takion, uint8_t base_type, uint8_t *buf, size_t buf_size)
+{
+	// HHIxxxxxIx
+
+	if(buf_size < AV_HEADER_SIZE + 1)
+	{
+		CHIAKI_LOGE(takion->log, "Takion received AV packet smaller than av header size + 1\n");
+		return;
+	}
+
+	uint16_t word_0 = ntohs(*((uint16_t *)(buf + 0)));
+	uint16_t word_1 = ntohs(*((uint16_t *)(buf + 2)));
+	uint32_t dword_2 = ntohl(*((uint32_t *)(buf + 4)));
+	uint32_t key_pos = ntohl(*((uint32_t *)(buf + 0xd)));
+	uint8_t unknown_1 = buf[0x11];
+
+	uint8_t *data = buf + AV_HEADER_SIZE;
+	size_t data_size = buf_size - AV_HEADER_SIZE;
+
+	if(takion->av_cb)
+		takion->av_cb(data, data_size, key_pos, takion->av_cb_user);
 }