Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Firefly 1608 redis messaging caching #1685

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

Firefly 1608 redis messaging caching #1685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5d90345
FIREFLY-1608: Switch to Redis for Caching and Messaging
loitly Nov 28, 2024
c597d76
convert events to redis sub/pub
loitly Dec 21, 2024
266e671
- preinstall DuckDB magic extension in the docker image
loitly Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions buildScript/dependencies.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ dependencies {

implementation ('com.googlecode.json-simple:json-simple:1.1.1') { transitive = false }

// jedis
implementation ('redis.clients:jedis:3.1.0') { exclude group: 'org.slf4j' }
// jedis; support redis versions: 5.0 to 7.2 Family of releases
implementation ('redis.clients:jedis:4.4.8') { exclude group: 'org.slf4j' }

// ehcache. exclude core because it contains duplicated older CacheManager
implementation ('net.sf.ehcache:ehcache:2.7.4')
Expand Down Expand Up @@ -98,6 +98,9 @@ dependencies {

// duckdb - SQL support for parquet, csv, and tsv files
implementation 'org.duckdb:duckdb_jdbc:1.1.3'

// embedded redis server; version ~ 6.2
implementation 'com.github.codemonstur:embedded-redis:1.4.3'
}


Expand Down
117 changes: 13 additions & 104 deletions config/ehcache.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="ehcache.xsd" name="@app-name@">
xsi:noNamespaceSchemaLocation="ehcache.xsd" name="localCache">

<!--
CacheManager Configuration
Expand Down Expand Up @@ -118,15 +118,6 @@
use 239.250.0.1 for org-local
use 230.0.0.1 for global
-->
<!--REPLICATE
<cacheManagerPeerProviderFactory
class="edu.caltech.ipac.util.cache.CachePeerProviderFactory"
properties="[email protected]@,
[email protected]@,
[email protected]@, [email protected]@"
propertySeparator=","
/>
REPLICATE-->

<!--
CacheManagerPeerListener
Expand Down Expand Up @@ -163,13 +154,6 @@
If not specified it defaults 120000ms.
-->

<!--REPLICATE
<cacheManagerPeerListenerFactory
class="net.sf.ehcache.distribution.RMICacheManagerPeerListenerFactory"
properties="socketTimeoutMillis=300000"
propertySeparator="," />
REPLICATE-->

<!--
Cache configuration
===================
Expand Down Expand Up @@ -372,106 +356,31 @@
-->

<!--
The data stored in this cache is relatively small and have a short idle time.
THIS CACHE IS DISTRIBUTED.
The data store in this cache is large and are meant to not exceed a specified max memory allocation.
This cache is shared among all web applications in the container. it will NOT be replicated.
The maxBytesLocalHeap is not important here because we will automatically calculate it based on
pct.vis.shared.mem.size property.
-->
<cache name="HTTP_SESSION"
maxElementsInMemory="200"
<cache name="VIS_SHARED_MEM"
maxBytesLocalHeap="100M"
eternal="false"
timeToIdleSeconds="1800"
memoryStoreEvictionPolicy="LRU"
overflowToDisk="true"
maxElementsOnDisk="0"
diskSpoolBufferSizeMB="5"
diskExpiryThreadIntervalSeconds="300"
>

<!--REPLICATE
<cacheEventListenerFactory
class="net.sf.ehcache.distribution.RMICacheReplicatorFactory"
properties="asynchronousReplicationIntervalMillis=250,
replicateAsynchronously=true"
propertySeparator=","/>
REPLICATE-->

</cache>
overflowToDisk="false"
/>

<!--
The data stored in this cache is relatively small, but does not expire.
THIS CACHE IS DISTRIBUTED.
The data in this cache is relatively small and does not expire by default.
However, if lifespanInSecs is set, objects exceeding the specified lifespan will be cleared.
-->
<cache name="PERM_SMALL"
maxElementsInMemory="1000"
eternal="true"
memoryStoreEvictionPolicy="LRU"
overflowToDisk="true"
diskPersistent="true"
maxElementsOnDisk="50000"
diskSpoolBufferSizeMB="5"
diskExpiryThreadIntervalSeconds="600"
>

<!--REPLICATE
<cacheEventListenerFactory
class="net.sf.ehcache.distribution.RMICacheReplicatorFactory"
properties="asynchronousReplicationIntervalMillis=250,
replicateAsynchronously=true"
propertySeparator=","/>
REPLICATE-->
</cache>

<!--
This cache is reserved for file path(File) caching. There is no idle time expiration.
This is similar to PERM_SMALL, except there will be a CacheEventListener to monitor
the File object, and to clean up the file when the File object is evicted.
THIS CACHE IS DISTRIBUTED.
-->
<cache name="PERM_FILE"
maxElementsInMemory="5000"
eternal="true"
eternal="false"
memoryStoreEvictionPolicy="LRU"
overflowToDisk="true"
diskPersistent="true"
maxElementsOnDisk="50000"
diskSpoolBufferSizeMB="5"
diskExpiryThreadIntervalSeconds="600"
>

<!--REPLICATE
<cacheEventListenerFactory
class="net.sf.ehcache.distribution.RMICacheReplicatorFactory"
properties="asynchronousReplicationIntervalMillis=250,
replicateAsynchronously=true"
propertySeparator=","/>
/>
REPLICATE-->

</cache>


<!--
This cache is reserved for file path(File) caching. There is no idle time expiration.
This is similar to PERM_SMALL, except there will be a CacheEventListener to monitor
the File object, and to clean up the file when the File object is evicted.
THIS CACHE IS DISTRIBUTED.
-->
<cache name="TEMP_FILE"
maxElementsInMemory="5000"
eternal="true"
memoryStoreEvictionPolicy="LRU"
overflowToDisk="false"
diskPersistent="false"
>

<!--REPLICATE
<cacheEventListenerFactory
class="net.sf.ehcache.distribution.RMICacheReplicatorFactory"
properties="asynchronousReplicationIntervalMillis=250,
replicateAsynchronously=true"
propertySeparator=","/>
/>
REPLICATE-->

</cache>
/>

</ehcache>
Loading