4 Replies Latest reply: Jun 4, 2012 6:32 AM by Dan Berindei RSS

Uneven data distribution across the cluster with virtual nodes

nit wit Newbie

Hi,

 

We are using Infinispan5.1.2.Final and we are still experiencing significantly un-even distribution of data across the cluster even after using virtual nodes as specified in the documentation  - https://docs.jboss.org/author/display/ISPN/Clustering+modes#Clusteringmodes-VirtualNodesImprovingthedistributionofdata

 

We have about 320 odd keys and we went upto 500 virtual nodes but we still saw keys distributed as (160, 80, 80) on a 3-node cluster.

 

Any pointers on how we can further improve distibution would be greatly appreciated.

 

Thanks.

 

Following is how our configuration looks:

 

tns:namedCacheConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">

    <tns:locking writeSkewCheck="false" useLockStriping="false" lockAcquisitionTimeout="300000" isolationLevel="READ_COMMITTED" concurrencyLevel="5000"/>

    <tns:loaders shared="false" preload="false" passivation="false"/>

    <tns:transaction useSynchronization="false" useEagerLocking="false" use1PcForAutoCommitTransactions="false" transactionMode="TRANSACTIONAL" syncRollbackPhase="true" syncCommitPhase="true" lockingMode="OPTIMISTIC" eagerLockSingleNode="false" cacheStopTimeout="30000" autoCommit="true">

        <tns:recovery recoveryInfoCacheName="__recoveryInfoCacheName__" enabled="false"/>

    </tns:transaction>

<tns:customInterceptors/>

    <tns:dataContainer class="org.infinispan.container.DefaultDataContainer">

<tns:properties/>

    </tns:dataContainer>

    <tns:eviction wakeUpInterval="-9223372036854775808" threadPolicy="DEFAULT" strategy="NONE" maxEntries="-1"/>

    <tns:expiration wakeUpInterval="-1" maxIdle="-1" lifespan="-1" reaperEnabled="true"/>

    <tns:unsafe unreliableReturnValues="false"/>

    <tns:clustering>

        <tns:sync replTimeout="300000"/>

<tns:stateRetrieval timeout="300000" retryWaitTimeIncreaseFactor="2" numRetries="10" maxNonProgressingLogWrites="100" logFlushTimeout="60000" initialRetryWaitTime="500" fetchInMemoryState="false" alwaysProvideInMemoryState="false">

<tns:chunkSize>1000</tns:chunkSize>

</tns:stateRetrieval>

        <tns:l1 onRehash="true" lifespan="60000" invalidationThreshold="0" enabled="true"/>

        <tns:async useReplQueue="false" replQueueMaxElements="1000" replQueueInterval="5000" replQueueClass="org.infinispan.remoting.ReplicationQueueImpl" asyncMarshalling="false"/>

        <tns:hash rehashWait="60000" rehashRpcTimeout="600000" rehashEnabled="true" numVirtualNodes="500" numOwners="1" hashFunctionClass="org.infinispan.commons.hash.MurmurHash3">

            <tns:groups enabled="true"/>

        </tns:hash>

    </tns:clustering>

    <tns:jmxStatistics enabled="true"/>

    <tns:storeAsBinary storeValuesAsBinary="true" storeKeysAsBinary="true" enabled="false"/>

<tns:lazyDeserialization enabled="false"/>

    <tns:deadlockDetection spinDuration="100" enabled="false"/>

    <tns:indexing indexLocalOnly="false" enabled="false">

<tns:properties/>

    </tns:indexing>

    <tns:versioning versioningScheme="NONE" enabled="false"/>

<tns:invocationBatching enabled="true"/>

</tns:namedCacheConfiguration>

 

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

<tns:globalConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">

<tns:asyncListenerExecutor factory="org.infinispan.executors.DefaultExecutorFactory">

<tns:properties/>

</tns:asyncListenerExecutor>

<tns:asyncTransportExecutor factory="org.infinispan.executors.DefaultExecutorFactory">

<tns:properties>

            <tns:property value="asyncTransportThread" name="threadNamePrefix"/>

</tns:properties>

</tns:asyncTransportExecutor>

<tns:evictionScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">

<tns:properties/>

</tns:evictionScheduledExecutor>

<tns:replicationQueueScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">

<tns:properties/>

</tns:replicationQueueScheduledExecutor>

<tns:globalJmxStatistics mBeanServerLookup="org.infinispan.jmx.PlatformMBeanServerLookup" jmxDomain="org.infinispan" enabled="true" cacheManagerName="DefaultCacheManager" allowDuplicateDomains="true">

<tns:properties/>

</tns:globalJmxStatistics>

    <tns:transport transportClass="org.infinispan.remoting.transport.jgroups.JGroupsTransport" strictPeerToPeer="false" distributedSyncTimeout="60000" clusterName="PERFpivotCluster">

<tns:properties>

            <tns:property value="10000" name="jgroups.merge2.min_interval"/>

            <tns:property value="true" name="jgroups.tcp.thread_pool.rejection_policy"/>

            <tns:property value="discard" name="jgroups.tcp.oob_thread_pool.rejection_policy"/>

            <tns:property value="discard" name="jgroups.tcp.discard_incompatible_packets"/>

            <tns:property value="15000" name="jgroups.fd_all.timeout"/>

            <tns:property value="500" name="jgroups.pbcast.gms.max_bundling_time"/>

            <tns:property value="10" name="jgroups.tcp.thread_pool.min_threads"/>

            <tns:property value="8" name="jgroups.tcp.oob_thread_pool.min_threads"/>

            <tns:property value="25" name="listener.maxThreads"/>

            <tns:property value="true" name="jgroups.barrier"/>

            <tns:property value="600,1200" name="jgroups.pbcast.nakack.retransmit_timeout"/>

            <tns:property value="300,600,1200" name="jgroups.pbcast.unicast.timeout"/>

            <tns:property value="true" name="jgroups.pbcast.nakack.discard_delivered_msgs"/>

            <tns:property value="false" name="jgroups.tcp.oob_thread_pool.queue_enabled"/>

            <tns:property value="true" name="jgroups.pbcast.NAKACK.use_mcast_xmit"/>

            <tns:property value="3000" name="jgroups.pbcast.gms.join_timeout"/>

            <tns:property value="500000" name="jgroups.fc.max_credits"/>

            <tns:property value="1000000" name="jgroups.pbcast.stable.max_bytes"/>

            <tns:property value="jgroupsBigDataTcpStatic.xml" name="configurationFile"/>

            <tns:property value="640000" name="jgroups.tcp.send_buf_size"/>

            <tns:property value="true" name="jgroups.tcp.thread_pool.enabled"/>

            <tns:property value="5000" name="jgroups.ping.timeout"/>

            <tns:property value="50000" name="jgroups.pbcast.stable.desired_avg_gossip"/>

            <tns:property value="30" name="jgroups.tcp.port_range"/>

            <tns:property value="true" name="jgroups.tcp.thread_pool.queue_enabled"/>

            <tns:property value="1000" name="jgroups.pbcast.stable.stability_delay"/>

            <tns:property value="true" name="jgroups.pbcast.gms.view_bundling"/>

            <tns:property value="30000" name="jgroups.merge2.max_interval"/>

            <tns:property value="100" name="jgroups.tcp.oob_thread_pool.queue_max_size"/>

            <tns:property value="20000000" name="jgroups.tcp.recv_buf_size"/>

            <tns:property value="false" name="jgroups.tcp.loopback"/>

            <tns:property value="5000" name="jgroups.tcp.oob_thread_pool.keep_alive_time"/>

            <tns:property value="60000" name="jgroups.frag2.frag_size"/>

            <tns:property value="0" name="jgroups.pbcast.NAKACK.exponential_backoff"/>

            <tns:property value="true" name="jgroups.fdsock"/>

            <tns:property value="400" name="jgroups.tcp.oob_thread_pool.max_threads"/>

            <tns:property value="300" name="jgroups.tcp.thread_pool.max_threads"/>

            <tns:property value="10000" name="jgroups.tcp.thread_pool.queue_max_size"/>

            <tns:property value="5000" name="jgroups.fd_all.interval"/>

            <tns:property value="true" name="jgroups.pbcast.gms.print_local_addr"/>

            <tns:property value="true" name="jgroups.pbcast.streaming_state_transfer"/>

            <tns:property value="PERFpivotCluster" name="clusterName"/>

            <tns:property value="15000" name="jgroups.verify_suspect.timeout"/>

            <tns:property value="0" name="jgroups.pbcast.nakack.gc_lag"/>

            <tns:property value="10" name="jgroups.udp.tos"/>

            <tns:property value="4" name="jgroups.ping.num_initial_members"/>

            <tns:property value="25" name="transport.maxThreads"/>

            <tns:property value="0.20" name="jgroups.fc.min_threshold"/>

            <tns:property value="5000" name="jgroups.tcp.thread_pool.keep_alive_time"/>

</tns:properties>

    </tns:transport>

    <tns:serialization version="5.1" marshallerClass="org.infinispan.marshall.VersionAwareMarshaller">

<tns:advancedExternalizers/>

</tns:serialization>

    <tns:shutdown hookBehavior="DEFAULT"/>

  </tns:globalConfiguration>