Привет!
Вопрос про Spark. Есть датасет с json'ами.
В нём есть поле, типа dict, его ключи: 1,2,3,4... (очень много). У каждого ключа есть вложенная структура. Так вот эти вложенные структуры одинаковые. Можно ли как-то сделать аналог explode, но не для массива, а для такого словаря?
схема выглядит как-то так:
|-- survey_data: struct (nullable = true)
| |-- 127: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
| |-- 144: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- parent: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
| |-- 145: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- parent: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
| |-- 146: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- parent: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
| |-- 147: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- parent: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
| |-- 148: struct (nullable = true)
| | |-- answer: string (nullable = true)
| | |-- answer_id: long (nullable = true)
| | |-- id: long (nullable = true)
| | |-- parent: long (nullable = true)
| | |-- question: string (nullable = true)
| | |-- section_id: long (nullable = true)
| | |-- shown: boolean (nullable = true)
| | |-- type: string (nullable = true)
.....
