4 Replies Latest reply on Jun 4, 2012 6:32 AM by dan.berindei

    Uneven data distribution across the cluster with virtual nodes

    agnihere

      Hi,

       

      We are using Infinispan5.1.2.Final and we are still experiencing significantly un-even distribution of data across the cluster even after using virtual nodes as specified in the documentation  - https://docs.jboss.org/author/display/ISPN/Clustering+modes#Clusteringmodes-VirtualNodesImprovingthedistributionofdata

       

      We have about 320 odd keys and we went upto 500 virtual nodes but we still saw keys distributed as (160, 80, 80) on a 3-node cluster.

       

      Any pointers on how we can further improve distibution would be greatly appreciated.

       

      Thanks.

       

      Following is how our configuration looks:

       

      tns:namedCacheConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">

          <tns:locking writeSkewCheck="false" useLockStriping="false" lockAcquisitionTimeout="300000" isolationLevel="READ_COMMITTED" concurrencyLevel="5000"/>

          <tns:loaders shared="false" preload="false" passivation="false"/>

          <tns:transaction useSynchronization="false" useEagerLocking="false" use1PcForAutoCommitTransactions="false" transactionMode="TRANSACTIONAL" syncRollbackPhase="true" syncCommitPhase="true" lockingMode="OPTIMISTIC" eagerLockSingleNode="false" cacheStopTimeout="30000" autoCommit="true">

              <tns:recovery recoveryInfoCacheName="__recoveryInfoCacheName__" enabled="false"/>

          </tns:transaction>

      <tns:customInterceptors/>

          <tns:dataContainer class="org.infinispan.container.DefaultDataContainer">

      <tns:properties/>

          </tns:dataContainer>

          <tns:eviction wakeUpInterval="-9223372036854775808" threadPolicy="DEFAULT" strategy="NONE" maxEntries="-1"/>

          <tns:expiration wakeUpInterval="-1" maxIdle="-1" lifespan="-1" reaperEnabled="true"/>

          <tns:unsafe unreliableReturnValues="false"/>

          <tns:clustering>

              <tns:sync replTimeout="300000"/>

      <tns:stateRetrieval timeout="300000" retryWaitTimeIncreaseFactor="2" numRetries="10" maxNonProgressingLogWrites="100" logFlushTimeout="60000" initialRetryWaitTime="500" fetchInMemoryState="false" alwaysProvideInMemoryState="false">

      <tns:chunkSize>1000</tns:chunkSize>

      </tns:stateRetrieval>

              <tns:l1 onRehash="true" lifespan="60000" invalidationThreshold="0" enabled="true"/>

              <tns:async useReplQueue="false" replQueueMaxElements="1000" replQueueInterval="5000" replQueueClass="org.infinispan.remoting.ReplicationQueueImpl" asyncMarshalling="false"/>

              <tns:hash rehashWait="60000" rehashRpcTimeout="600000" rehashEnabled="true" numVirtualNodes="500" numOwners="1" hashFunctionClass="org.infinispan.commons.hash.MurmurHash3">

                  <tns:groups enabled="true"/>

              </tns:hash>

          </tns:clustering>

          <tns:jmxStatistics enabled="true"/>

          <tns:storeAsBinary storeValuesAsBinary="true" storeKeysAsBinary="true" enabled="false"/>

      <tns:lazyDeserialization enabled="false"/>

          <tns:deadlockDetection spinDuration="100" enabled="false"/>

          <tns:indexing indexLocalOnly="false" enabled="false">

      <tns:properties/>

          </tns:indexing>

          <tns:versioning versioningScheme="NONE" enabled="false"/>

      <tns:invocationBatching enabled="true"/>

      </tns:namedCacheConfiguration>

       

      <?xml version="1.0" encoding="UTF-8" standalone="yes"?>

      <tns:globalConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">

      <tns:asyncListenerExecutor factory="org.infinispan.executors.DefaultExecutorFactory">

      <tns:properties/>

      </tns:asyncListenerExecutor>

      <tns:asyncTransportExecutor factory="org.infinispan.executors.DefaultExecutorFactory">

      <tns:properties>

                  <tns:property value="asyncTransportThread" name="threadNamePrefix"/>

      </tns:properties>

      </tns:asyncTransportExecutor>

      <tns:evictionScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">

      <tns:properties/>

      </tns:evictionScheduledExecutor>

      <tns:replicationQueueScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">

      <tns:properties/>

      </tns:replicationQueueScheduledExecutor>

      <tns:globalJmxStatistics mBeanServerLookup="org.infinispan.jmx.PlatformMBeanServerLookup" jmxDomain="org.infinispan" enabled="true" cacheManagerName="DefaultCacheManager" allowDuplicateDomains="true">

      <tns:properties/>

      </tns:globalJmxStatistics>

          <tns:transport transportClass="org.infinispan.remoting.transport.jgroups.JGroupsTransport" strictPeerToPeer="false" distributedSyncTimeout="60000" clusterName="PERFpivotCluster">

      <tns:properties>

                  <tns:property value="10000" name="jgroups.merge2.min_interval"/>

                  <tns:property value="true" name="jgroups.tcp.thread_pool.rejection_policy"/>

                  <tns:property value="discard" name="jgroups.tcp.oob_thread_pool.rejection_policy"/>

                  <tns:property value="discard" name="jgroups.tcp.discard_incompatible_packets"/>

                  <tns:property value="15000" name="jgroups.fd_all.timeout"/>

                  <tns:property value="500" name="jgroups.pbcast.gms.max_bundling_time"/>

                  <tns:property value="10" name="jgroups.tcp.thread_pool.min_threads"/>

                  <tns:property value="8" name="jgroups.tcp.oob_thread_pool.min_threads"/>

                  <tns:property value="25" name="listener.maxThreads"/>

                  <tns:property value="true" name="jgroups.barrier"/>

                  <tns:property value="600,1200" name="jgroups.pbcast.nakack.retransmit_timeout"/>

                  <tns:property value="300,600,1200" name="jgroups.pbcast.unicast.timeout"/>

                  <tns:property value="true" name="jgroups.pbcast.nakack.discard_delivered_msgs"/>

                  <tns:property value="false" name="jgroups.tcp.oob_thread_pool.queue_enabled"/>

                  <tns:property value="true" name="jgroups.pbcast.NAKACK.use_mcast_xmit"/>

                  <tns:property value="3000" name="jgroups.pbcast.gms.join_timeout"/>

                  <tns:property value="500000" name="jgroups.fc.max_credits"/>

                  <tns:property value="1000000" name="jgroups.pbcast.stable.max_bytes"/>

                  <tns:property value="jgroupsBigDataTcpStatic.xml" name="configurationFile"/>

                  <tns:property value="640000" name="jgroups.tcp.send_buf_size"/>

                  <tns:property value="true" name="jgroups.tcp.thread_pool.enabled"/>

                  <tns:property value="5000" name="jgroups.ping.timeout"/>

                  <tns:property value="50000" name="jgroups.pbcast.stable.desired_avg_gossip"/>

                  <tns:property value="30" name="jgroups.tcp.port_range"/>

                  <tns:property value="true" name="jgroups.tcp.thread_pool.queue_enabled"/>

                  <tns:property value="1000" name="jgroups.pbcast.stable.stability_delay"/>

                  <tns:property value="true" name="jgroups.pbcast.gms.view_bundling"/>

                  <tns:property value="30000" name="jgroups.merge2.max_interval"/>

                  <tns:property value="100" name="jgroups.tcp.oob_thread_pool.queue_max_size"/>

                  <tns:property value="20000000" name="jgroups.tcp.recv_buf_size"/>

                  <tns:property value="false" name="jgroups.tcp.loopback"/>

                  <tns:property value="5000" name="jgroups.tcp.oob_thread_pool.keep_alive_time"/>

                  <tns:property value="60000" name="jgroups.frag2.frag_size"/>

                  <tns:property value="0" name="jgroups.pbcast.NAKACK.exponential_backoff"/>

                  <tns:property value="true" name="jgroups.fdsock"/>

                  <tns:property value="400" name="jgroups.tcp.oob_thread_pool.max_threads"/>

                  <tns:property value="300" name="jgroups.tcp.thread_pool.max_threads"/>

                  <tns:property value="10000" name="jgroups.tcp.thread_pool.queue_max_size"/>

                  <tns:property value="5000" name="jgroups.fd_all.interval"/>

                  <tns:property value="true" name="jgroups.pbcast.gms.print_local_addr"/>

                  <tns:property value="true" name="jgroups.pbcast.streaming_state_transfer"/>

                  <tns:property value="PERFpivotCluster" name="clusterName"/>

                  <tns:property value="15000" name="jgroups.verify_suspect.timeout"/>

                  <tns:property value="0" name="jgroups.pbcast.nakack.gc_lag"/>

                  <tns:property value="10" name="jgroups.udp.tos"/>

                  <tns:property value="4" name="jgroups.ping.num_initial_members"/>

                  <tns:property value="25" name="transport.maxThreads"/>

                  <tns:property value="0.20" name="jgroups.fc.min_threshold"/>

                  <tns:property value="5000" name="jgroups.tcp.thread_pool.keep_alive_time"/>

      </tns:properties>

          </tns:transport>

          <tns:serialization version="5.1" marshallerClass="org.infinispan.marshall.VersionAwareMarshaller">

      <tns:advancedExternalizers/>

      </tns:serialization>

          <tns:shutdown hookBehavior="DEFAULT"/>

        </tns:globalConfiguration>