segFault while write large data on multiple thread

[ChiragGangwani]

We are writing without write batch and system got crash

Stack: [0x00007f6fe09de000,0x00007f6fe0adf000], sp=0x00007f6fe0adb750, free space=1013k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C [librocksdbjni8579403097751306729.so+0x5f3eec] rocksdb::WritableFileWriter::Flush(rocksdb::Env::IOPriority)+0xfc
C [librocksdbjni8579403097751306729.so+0x58f1ac] rocksdb::EnvLogger::Flush()+0xec
C [librocksdbjni8579403097751306729.so+0x5fce80] rocksdb::AutoRollLogger::Flush()+0x60
C [librocksdbjni8579403097751306729.so+0x58656b]
C [librocksdbjni8579403097751306729.so+0x5895d4] rocksdb::Log(rocksdb::InfoLogLevel, std::shared_ptrrocksdb::Logger const&, char const*, ...)+0x84
C [librocksdbjni8579403097751306729.so+0x448c78] rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::string const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocatorrocksdb::ColumnFamilyDescriptor > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocatorrocksdb::ColumnFamilyHandle* >, rocksdb::DB**, bool, bool)+0x738
C [librocksdbjni8579403097751306729.so+0x44a7e6] rocksdb::DB::Open(rocksdb::DBOptions const&, std::string const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocatorrocksdb::ColumnFamilyDescriptor > const&, std::vector<rocksdb::ColumnFamilyHandle, std::allocatorrocksdb::ColumnFamilyHandle* >, rocksdb::DB**)+0x56
C [librocksdbjni8579403097751306729.so+0x44a996] rocksdb::DB::Open(rocksdb::Options const&, std::string const&, rocksdb::DB**)+0x176
C [librocksdbjni8579403097751306729.so+0x2d4aba] std::_Function_handler<rocksdb::Status (rocksdb::Options const&, std::string const&, rocksdb::DB**), rocksdb::Status ()(rocksdb::Options const&, std::string const&, rocksdb::DB**)>::_M_invoke(std::Any_data const&, rocksdb::Options const&, std::string const&, rocksdb::DB**&&)+0x1a
C [librocksdbjni8579403097751306729.so+0x2cf156] rocksdb_open_helper(JNIEnv, long, _jstring, std::function<rocksdb::Status (rocksdb::Options const&, std::string const&, rocksdb::DB**)>)+0x86
C [librocksdbjni8579403097751306729.so+0x2cf25e] Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2+0x3e
j org.rocksdb.RocksDB.open(JLjava/lang/String;)J+0
j org.rocksdb.RocksDB.open(Lorg/rocksdb/Options;Ljava/lang/String;)Lorg/rocksdb/RocksDB;+9
j

[alanpaxton]

Hi @ChiragGangwani If you could provide further details of what you are doing, with a reproduction if possible, plus platform and RocksDB version you are using, it would make it more possible for someone to understand what might be going wrong..

[ChiragGangwani]

Hi @alanpaxton thanks for the response. Please find the further deatils below.

RocksDB version - 8.5.4

So we are doing high amount of write in multiple db & column families in multiple thread with write batch functionality.
There is no reproduction code because it happen unexpected anytime but only with large data.

write batch size - 50 mb.
Here some configuration -

DB Creation code -

private void initDB() throws StoreInitException {
        Logger.info("MapStore write batch size - " + storeProps.getWriteBatchSize());
        this.options.setStatistics(new Statistics());
        this.initCache();
        this.options.setAllowMmapReads(true);
        this.options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
        this.options.setMaxLogFileSize(1000);
        this.options.setKeepLogFileNum(1);
        if (storeProps.isRead()) {
            createDescriptors();
            try
            {
                this.db = RocksDB.openReadOnly(dbOptions, this.storeProps.getDbFolder(), descriptors,
                        columnFamilyHandles);
            }
            catch (RocksDBException e)
            {
                throw new StoreInitException(e);
            }
            createDescriptorHandleMap();
            try {
                this.mapStoreMetadata = MapStoreMetadata.deserializeMetadata(storeProps.getDbFolder());
            } catch (Exception e) {
                Logger.warn("Store metadata is not available, metadata operations might be slow.");
                Logger.warn(e);
            }
        }
        else
        {
            Utility.ensureEmptyFolder(this.storeProps.getDbFolder());
            this.options.setCreateIfMissing(true);
            this.options.setCreateMissingColumnFamilies(true);
            try
            {
                this.db = RocksDB.open(options, this.storeProps.getDbFolder());
            }
            catch (RocksDBException e)
            {
                Logger.error(e);
                throw new StoreInitException(e);
            }
            addSegments(storeProps.getSegmentList());
        }
        defaultColumnFamily = this.db.getDefaultColumnFamily();
        logInitializingStore();
    }

Cache -

synchronized private void initCache() {
      storeProps.getCacheProperty().ifPresent(cacheProperties -> {
          Function<String, Cache> cacheFunc = id -> CacheType.CLOCK.equals(cacheProperties.getCacheType())
                  ? new ClockCache(cacheProperties.getCacheSize())
                  : new LRUCache(cacheProperties.getCacheSize());

          Cache cache = cacheMap.computeIfAbsent(cacheProperties.getId(), cacheFunc);
          this.blockBasedTableConfig = new BlockBasedTableConfig().setBlockCache(cache);
          this.options.setWriteBufferSize(cacheProperties.getWriteBufferSize());
          this.options.setTableFormatConfig(blockBasedTableConfig);
      });
  }

Put Operation -

public void put(Optional<String> segmentId, K key, V val) throws StoreWriteException {
        byte[] keyArr = toByteArr(key);
        byte[] valArr = toByteArr(val);
            try {
                **if(storeProps.getWriteBatchSize()>0){
                    putInBatch(segmentId, key, val);
                }**
                else {
                    this.db.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
                }
            } catch (RocksDBException e) {
                Logger.error(e);
                throw new StoreWriteException(e);
            }
    }

private void putInBatch(Optional<String> segmentId, K key, V val) throws StoreWriteException {
        byte[] keyArr = toByteArr(key);
        byte[] valArr = toByteArr(val);
           try {
               writeBatch.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
           } catch (RocksDBException e) {
               throw new StoreWriteException(e);
           }
        if (writeBatch.getDataSize() >= storeProps.getWriteBatchSize()) {
            putBatchInDb();
        }
    }

private void putBatchInDb() throws StoreWriteException {
     try {
         if(writeBatch.count()>0){
             db.write(new WriteOptions(), writeBatch);
             writeBatch.clear();
         }
     } catch (RocksDBException e) {
         throw new StoreWriteException(e);
     }
 }

[ChiragGangwani]

Uploading hs_err_pid3859541.log…
Please check the crash logs for further details.