[go-nuts] Efficient data storage of unknown struct types

Hi all. I'm trying to find an efficient (GC, mem footprint and speed) way
to create a storage package that can accept different struct types, create
a bucket for each type and then recycle the memory as objects are deleted
and added (e.g. reuse slots). It's designed to use offsets and also
optionally allows linking of objects (note the bucket is not one big list,
but rather can contain many smaller lists).

The motivation was unacceptable GC pause with large mem footprints of map
-> pointers -> objects.

So my questions are:
1. Is below on the right track?
2. I can't work out how to write an object at a given slice offset? Below
does not work (see Add() function). I get that i need to reflect the
interface somehow, but haven't managed to find a way.
3. From reading, many people say the reflection overhead is expensive, so
is this not worth doing?

Many thanks.
Hamish


*package main*
*
*
*import (*
* "fmt"*
* "reflect"*
*)*
*
*
*
*
*type StorageEngine struct {*
* Buckets map[uint16]Bucket // Max 65,535 buckets*
* NoOfBuckets uint16 *
*}*
*
*
*// Basic structure of a data bucket. Next and Prev are optional*
*type Bucket struct {*
* ID uint16 // Unique ID for each bucket*
* Datatype string // Description only*
* Length uint32 // Max number of objects 4,294,967,295*
* Offset uint32 // Current write position. Once Offset=Length, only the
DeletedSlots can be used or a new bucket must be created*
* Data interface{} // Any struct can be used as the data type, once created
it cannot change*
* Next []uint32 // Optional offset for the next record in a linked list*
* Prev []uint32 // Optional offset for the previous record in a linked list*
* DeletedSlots chan uint32 // Channel reuses slots when they are deleted*
*}*
*
*
*
*
*// Initialise a new storage engine*
*func (s *StorageEngine) Init() {*
* s.Buckets = make(map[uint16]Bucket, 100)*
* s.NoOfBuckets = 0*
*}*
*
*
*// Create a new bucket of type datastruct*
*func (s *StorageEngine) NewBucket(datatype string, length uint32, next
bool, prev bool, datastruct interface{}) uint16 {*
* b := new(Bucket)*
* b.ID = s.ReqNewBucketID()*
* b.Datatype = datatype*
* b.DeletedSlots = make(chan uint32, length)*
* myType := reflect.TypeOf(datastruct)*
* b.Data = reflect.MakeSlice(reflect.SliceOf(myType), int(length),
int(length)).Interface()*
* if next {*
* b.Next = make([]uint32, length)*
* }*
* if prev {*
* b.Prev = make([]uint32, length)*
* }*
* return b.ID*
*}*
*
*
*// Allocate and return a new bucket ID*
*func (s *StorageEngine) ReqNewBucketID() uint16 {*
* count := s.NoOfBuckets*
* s.NoOfBuckets++*
* return count*
*}*
*
*
*// Add a new data object to a bucket*
*func (s *StorageEngine) Add(id uint16, object interface{}, next uint32,
prev uint32) bool {*
* bucket := s.Buckets[id]*
* // If there is a deleted slot, use it, otherwise write to the offset*
* var slot uint32*
* select {*
* case slot = <- bucket.DeletedSlots:*
*
*
* default:*
* slot = bucket.Offset*
* }*
* data := reflect.TypeOf(bucket.Data)*
* data[slot] = object*
* bucket.Offset++*
* return true*
*}*
*
*
*
*
*
*
*type Cat struct {*
* ID int*
* Name string*
*}*
*
*
*type Dog struct {*
* ID int*
* Name string*
* Breed string*
*}*
*
*
*func main() {*
* store := new(StorageEngine)*
* store.Init()*
* x := new(Cat)*
* id := store.NewBucket("term", 100000, true, true, x)*
* fmt.Println("New bucket with ID = ",id)*
* x.ID = 1*
* x.Name = "whatever"*
* store.Add(id, x, 0, 0)*
*
*
*}*