Pi4J – ADC MCP3008 – SPI – Sensor Reader Example


import com.pi4j.io.spi.SpiChannel;
import com.pi4j.io.spi.SpiDevice;
import com.pi4j.io.spi.SpiFactory;

import java.nio.ByteBuffer;
import java.io.IOException;

public class SpiMCP3008 {

    public static SpiDevice spi = null;
    public static byte INIT_CMD = (byte) 0xD0; // 11010000

    public static void main(String args[]) throws InterruptedException, IOException {
        System.out.println("<--Pi4J--> SPI test program using MCP3008 AtoD Chip");

        spi = SpiFactory.getInstance(SpiChannel.CS0,
                                     SpiDevice.DEFAULT_SPI_SPEED, // default spi speed 1 MHz
                                     SpiDevice.DEFAULT_SPI_MODE); // default spi mode 0

        while(true) {
            read(0); // Read channel 1
			read(1); // Read channel 2
            Thread.sleep(100);
        }
    }

    public static void read(int channel) throws IOException {
        // 10-bit ADC MCP3008
        byte packet[] = new byte[3];
        packet[0] = 0x01;  // INIT_CMD;  // address byte
        packet[1] = (byte) ((0x08 + channel) << 4);  // singleEnded + channel
        packet[2] = 0x00;
           
        byte[] result = spi.write(packet);
        System.out.println( ((result[1] & 0x03 ) << 8) | (result[2] & 0xff) );
    }
}

Pi4J – DAC MCP4725 – I2C Example – Variable Voltage for a SyRen Motor Controller


import com.pi4j.io.i2c.I2CBus;
import com.pi4j.io.i2c.I2CDevice;
import com.pi4j.io.i2c.I2CFactory;

import java.io.IOException;

public class I2CTest {

    public static void main(String[] args) throws Exception {
        System.out.println("Starting:");

        final I2CBus bus = I2CFactory.getInstance(I2CBus.BUS_1);

        MotorController motorController = new MotorController(bus);
        motorController.init();

        while (true) {
	    motorController.write(1024);
            Thread.sleep(5000);
            motorController.write(2048);
            Thread.sleep(5000);
            motorController.write(3064);
            Thread.sleep(5000);
        }

    }

    public static class MotorController {

        private I2CDevice device;

	int REG_WRITEDAC = 0x40;
        int REG_WRITEDACEEPROM = 0x60;

        public MotorController(I2CBus bus) throws IOException {
            device = bus.getDevice(0x62); // MCP4725
        }

        public void write(int voltage) throws IOException {
            // 12-bit DAC
	    if (voltage > 4095) {
                voltage = 4095;
            }
            if (voltage < 0) {
                voltage = 0;
            }
            System.out.println("Setting voltage to: " + voltage);
            // Value needs to be left-shifted four bytes for the MCP4725
            byte[] bytes = {(byte)((voltage >> 4) & 0xFF),(byte)((voltage << 4) & 0xFF)};
            device.write(REG_WRITEDACEEPROM, bytes, 0, 2);
        }

        public int read() throws IOException {
            byte[] buf = new byte[256];
            int res = device.read(0, buf, 0, 6);
            return 0;
        }

    }

}

Throttled Producers and Consumers in Java

BlockingQueue queue = new ArrayBlockingQueue<>(QUEUE_MAXIMUM_SIZE);

producers:  queue.put(dbObject);

consumers: while (RUNNING) {  DBObject dbObject = queue.poll(1, TimeUnit.SECONDS); ...

When transferring large volumes of data from one type of database (Mongo for example), to another (DynamoDB), the fastest way is to do things in parallel. Reading from one Mongo cursor from one Mongo server in the cluster gives you limited performance. Writing to DynamoDB on one thread maxes out quickly to something like 100/s. I ended up creating many threads to read from Mongo in parallel on defined partitions to utilize all of the shards and replica sets in the cluster. I had these threads put items on the queue. Setting a max size on the BlockingQueue would throttle my reads to not flood the consumers and run out of memory. The consumers would read from the queue and eventually be told to stop when all of the producers were done. This model was very performant and easy to monitor. By monitoring the size of the queue, I could see if the consumers or producers were working faster. With the BlockingQueue, the producers were not allowed to out-pace the consumers. The consumers didn’t have any problems with this though since DynamoDB is way more scalable than Mongo.

AWS DynamoDB map object to Base64 encoded gzipped JSON in Java


Annotation in a DynamoDBTable class:
    @DynamoDBAttribute
    @DynamoDBMarshalling(marshallerClass=PojoMarshaller.class)
    private Pojo pojo;

public static class PojoMarshaller extends GzipJsonMarshaller<Pojo> { }

-------------------------------------------------------------------------------------------------------------

import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMarshaller;
import com.fasterxml.jackson.databind.*;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;

import java.io.*;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.zip.*;

import static com.amazonaws.util.Throwables.failure;

public class GzipJsonMarshaller<T extends Object> implements DynamoDBMarshaller<T> {

    private static final ObjectMapper mapper = new ObjectMapper();
    private static final ObjectWriter writer = mapper.writer();

    @Override
    public String marshall(T obj) {
        try {
            String plainJsonString = writer.writeValueAsString(obj);
            byte[] binaryBytes = compressString(plainJsonString).array();
            String base64BinaryString = Base64.encodeBase64String(binaryBytes);
            return base64BinaryString;
        } catch (Exception e) {
            throw failure(e, "Unable to marshall the instance of " + obj.getClass() + "into a string");
        }
    }

    @Override
    public T unmarshall(Class<T> clazz, String base64BinaryString) {
        try {
            byte[] binaryBytes = Base64.decodeBase64(base64BinaryString);
            String plainJsonString = uncompressString(ByteBuffer.wrap(binaryBytes));
            return mapper.readValue(plainJsonString, clazz);
        } catch (Exception e) {
            throw failure(e, "Unable to unmarshall the string " + base64BinaryString + "into " + clazz);
        }
    }

    public static ByteBuffer compressString(String input) throws IOException {
        ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
        GZIPOutputStream gzipOutput = new GZIPOutputStream(byteArrayOutput);
        gzipOutput.write(input.getBytes("UTF-8"));
        gzipOutput.finish();
        byte[] compressedBytes = byteArrayOutput.toByteArray();
        ByteBuffer buffer = ByteBuffer.wrap(compressedBytes);
        return buffer;
    }

    public static String uncompressString(ByteBuffer input) throws IOException {
        byte[] bytes = input.array();
        ByteArrayInputStream byteArrayInput = new ByteArrayInputStream(bytes);
        GZIPInputStream gzipInput = new GZIPInputStream(byteArrayInput);
        ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
        IOUtils.copy(gzipInput, byteArrayOutput);
        return new String(byteArrayOutput.toByteArray(), "UTF-8");
    }

}


Convert Mongo DBObject to POJO with Jongo


import org.jongo.ResultHandler;
import org.jongo.bson.Bson;
import org.jongo.bson.BsonDocument;
import org.jongo.marshall.Unmarshaller;
import org.jongo.marshall.jackson.JacksonEngine;
import org.jongo.marshall.jackson.configuration.Mapping;

...

    public Pojo getPojo(DBObject dbObject) {
        JacksonEngine engine = new JacksonEngine(new Mapping.Builder().build());
        ResultHandler<Pojo> handler = new UnmarshallingResultHandler<>(engine, Pojo.class); 
        Pojo pojo = handler.map(dbObject);
        return pojo;
    }

    public static class UnmarshallingResultHandler<T> implements ResultHandler<T> {
        private final Unmarshaller unmarshaller;
        private final Class<T> clazz;
        public UnmarshallingResultHandler(Unmarshaller unmarshaller, Class<T> clazz) {
            this.unmarshaller = unmarshaller;
            this.clazz = clazz;
        }
        public T map(DBObject result) {
            BsonDocument bsonDocument = Bson.createDocument(result);
            return unmarshaller.unmarshall(bsonDocument, clazz);
        }
    }

Apache HTTP Client 4.3.4 Example with Timeouts, Client Cert and Trusted Cert from KeyStores

If you have ever used or tried to use the Apache HTTP Client library, you’ve probably discovered that is is very useful and powerful, but they change the library so much each version that it is hard to find proper examples of how to do complex things like adding connection timeouts, basic auth, or adding a client cert, or trusting a self-signed cert. Here’s a code snippet of getting a client cert and a trusted server cert loaded with timeouts in version 4.3.4 of the library.

    <dependency>
	<groupId>org.apache.httpcomponents</groupId>
	<artifactId>httpclient</artifactId>
	<version>4.3.4</version>
    </dependency>

    public static HttpClient getSSLClient(String serverKeystoreFile, String serverKeystorePassword, String serverKeystoreType,
                                         String clientCertFile, String clientCertPassword, String clientKeystoreType) throws Exception {

        // Server cert trust stuff
        KeyStore trustStore = KeyStore.getInstance(serverKeystoreType);
        trustStore.load(new FileInputStream(new File(serverKeystoreFile)), serverKeystorePassword.toCharArray());
        TrustManagerFactory trustFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
        trustFactory.init(trustStore);
        TrustManager[] trustManagers = trustFactory.getTrustManagers();

        // Client cert stuff
        KeyStore clientCert = KeyStore.getInstance(clientKeystoreType);
        clientCert.load(new FileInputStream(new File(clientCertFile)), clientCertPassword.toCharArray());
        KeyManagerFactory keyFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
        keyFactory.init(clientCert, clientCertPassword.toCharArray());
        KeyManager[] keyManagers = keyFactory.getKeyManagers();

        SSLContext sslcontext = SSLContext.getInstance("TLS");
        sslcontext.init(keyManagers, trustManagers, new SecureRandom());

        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext);

        HttpClient httpClient = HttpClients.custom()
                .setSSLSocketFactory(sslsf)
                .build();

        return httpClient;
    }

    public static String executePost() throws Exception {

	 URI uri = new URIBuilder("http://my.superawesome.fakeurlstuffandstuff.com/path1/path2/file.php")).build();
	 String serverKeystore = Properties.get("TRUSTED_CERT_KEYSTORE_FILE_PATH"); // JKS FILE
	 String serverPass = Properties.get("TRUSTED_CERT_KEYSTORE_PASSWORD");
	 String clientKeystore = Properties.get("CLIENT_KEYSTORE"); // PFX FILE
	 String clientPass = Properties.get("CLIENT_KEYSTORE_PASSWORD");

	 HttpClient client = HttpJsonUtil.getClient(serverKeystore, serverPass, "JKS", clientKeystore, clientPass, "PKCS12");

	 RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
	 requestConfigBuilder.setMaxRedirects(2);
	 requestConfigBuilder.setConnectionRequestTimeout(Properties.getInt("SERVER_REQUEST_TIMEOUT_MILLIS", 30_000));
	 requestConfigBuilder.setConnectTimeout(Properties.getInt("SERVER_CONNECT_TIMEOUT_MILLIS", 10_000));

	 httpPost = new HttpPost(uri);
	 httpPost.setConfig(requestConfigBuilder.build());
	 httpPost.setEntity(new StringEntity("REQUEST BODY"));
	 httpPost.setHeader("Content-Type", "application/json");
	 httpPost.setHeader("Accept", "application/json");
	 HttpResponse response = client.execute(httpPost);
	 return EntityUtils.toString(response.getEntity());       
    }

Java TAXII Server Implementation

TAXII is a protocol standard for sharing security threat information between systems.  TAXII operates over HTTP/HTTPS and uses a specific XML schema for the message payloads.  Typically STIX is used as the main content of the messages.  TAXII is slowly being adopted by various organizations, particularly the banking industry.  The folks backing TAXII have provided YETI, a basic Python/Django TAXII server implementation and a Python client as well.  They also have a Java client which provides all the JAXB XML mappings for the specification.  In most enterprise environments, Python will not be ideal.  I decided to take the TAXII Java Client code and produce a simple Java Jersey (JAX-RS) TAXII server implementation of the DISCOVERY and POLL endpoints only. There are lots of println’s to show what is going on.  I built the java-taxii  and stix-binding code into a jars and loaded them into a local Maven repo to have available.

Dependencies: com.sun.jersey:jersey-bundle:1.17, javax.servlet:servlet-api:2.4, org.mitre.stix:stix-bindings:1.0 and org.mitre.taxii:java-taxii-all:1.0

Avalanche/SoltraEdge is a nice implementation of TAXII with a great UI, but licensing of the community edition is very limited in terms of usage and customization/modification.  It is developed in Python/Django as well and is most likely a spin-off from YETI.  Their website is incomplete, so it is hard to know what their commercial offerings are.  If you already have your security threat data stored somewhere, you’ll want to be able to customize your own TAXII interface in front of this data store.  Even YETI assumes you want to store incoming data in its own SQLITE database and serve it up from there.

The protocol is well documented, so it is easy to understand what each field is for so you can abide by the specifications.  The INBOX and COLLECTION_MANAGEMENT endpoints are more complex than the POLL and DISCOVERY endpoints, but can be done in a similar fashion.

Follow-up post with a sample/starter collection management endpoint: Java Taxii Collection Management Endpoint

import java.io.*;
import java.math.BigInteger;
import java.net.URI;
import java.util.*;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.xml.bind.*;
import javax.xml.datatype.*;
import org.mitre.taxii.*;
import org.mitre.taxii.client.HttpClient;
import org.mitre.taxii.messages.TaxiiXml;
import org.mitre.taxii.messages.xml11.*;

@Path("/")
public class TestEndpoints {

    /* Example request headers
            User-Agent: java-taxii.httpclient
            content-type: application/xml
            accept: application/xml
            x-taxii-accept: urn:taxii.mitre.org:message:xml:1.1
            x-taxii-content-type: urn:taxii.mitre.org:message:xml:1.1
            x-taxii-services: urn:taxii.mitre.org:services:1.1
            x-taxii-protocol: urn:taxii.mitre.org:protocol:http:1.0
    */
    
    // Are these thread-safe???
    private ObjectFactory factory = new ObjectFactory();
    private TaxiiXmlFactory txf = new TaxiiXmlFactory();
    private TaxiiXml taxiiXml = txf.createTaxiiXml();
    
    @POST
    @Path("discovery")
    @Produces (MediaType.APPLICATION_XML)
    @Consumes (MediaType.APPLICATION_XML)
    public Response discovery(@Context HttpServletRequest request, String x) {
        try {
            printHeaders(request);

            System.out.println("---------- Request:");
            DiscoveryRequest discoveryRequest = (DiscoveryRequest) getRequestObject(x);
            System.out.println(toXml(discoveryRequest));

            System.out.println("---------- Response:");
            List services = new ArrayList<>();
            services.add(factory.createServiceInstanceType()
                    .withServiceType(ServiceTypeEnum.POLL)
                    .withAddress("/poll")
                    .withAvailable(true)
                    .withProtocolBinding(Versions.VID_TAXII_HTTP_10)
                    .withServiceVersion(Versions.VID_TAXII_SERVICES_11)
                    .withMessageBindings(Versions.VID_TAXII_XML_11)
                    .withMessage("Super awesome data comes from this service")
                    .withContentBindings(factory.createContentBindingIDType().withBindingId(ContentBindings.CB_STIX_XML_111))
            );
            
            DiscoveryResponse discoveryResponse = factory.createDiscoveryResponse()
                    .withInResponseTo(discoveryRequest.getMessageId())
                    .withMessageId(MessageHelper.generateMessageId())
                    .withServiceInstances(services);
            
            String responseString = toXml(discoveryResponse);
            System.out.println(taxiiXml.marshalToString(discoveryResponse, true));
            
            return generateResponse(responseString, request);
        } catch(Exception ex) {
            ex.printStackTrace();
            return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
        }
    }
    
    @POST
    @Path("poll")
    @Produces (MediaType.APPLICATION_XML)
    @Consumes (MediaType.APPLICATION_XML)
    public Response poll(@Context HttpServletRequest request, String x) {
        try {
            printHeaders(request);

            System.out.println("---------- Request:");
            PollRequest pollRequest = (PollRequest) getRequestObject(x);
            System.out.println(toXml(pollRequest));
            String type = pollRequest.getPollParameters().getResponseType().equals(ResponseTypeEnum.FULL) ? "FULL" : "COUNT ONLY";
            System.out.println("Response Type: " + type);
            System.out.println("Collection: " + pollRequest.getCollectionName());
            System.out.println("Start Time: " + (pollRequest.getExclusiveBeginTimestamp() != null ? pollRequest.getExclusiveBeginTimestamp().toXMLFormat() : "(none)"));
            System.out.println("End Time: " + (pollRequest.getInclusiveEndTimestamp() != null ? pollRequest.getInclusiveEndTimestamp().toXMLFormat() : "(none)"));

            System.out.println("---------- Response:");
            PollResponse pollResponse = factory.createPollResponse()
                    .withInResponseTo(pollRequest.getMessageId())
                    .withMessageId(MessageHelper.generateMessageId())
                    .withCollectionName(pollRequest.getCollectionName())
                    .withRecordCount(factory.createRecordCountType().withValue(BigInteger.valueOf(9999)).withPartialCount(false)) 
                    .withExclusiveBeginTimestamp(pollRequest.getExclusiveBeginTimestamp())
                    .withInclusiveEndTimestamp(pollRequest.getInclusiveEndTimestamp())
                    .withContentBlocks(
                            factory.createContentBlock()
                                    .withContentBinding(factory.createContentInstanceType().withBindingId(ContentBindings.CB_STIX_XML_111))
                                    .withContent(factory.createAnyMixedContentType().withContent("Content Block Stuff Goes Here, STIX for example"))
                                    .withTimestampLabel(getTimestamp(null))
                                    .withMessage("Here's your data")
                    );
            
            String responseString = toXml(pollResponse);
            System.out.println(taxiiXml.marshalToString(pollResponse, true));
            
            return generateResponse(responseString, request);
        } catch(Exception ex) {
            ex.printStackTrace();
            return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
        }
    }
    
    private Response generateResponse(String responseString, HttpServletRequest request) throws Exception {
        return Response.ok(responseString)
                    .header(HttpClient.HEADER_X_TAXII_PROTOCOL, getProtocol(request))
                    .header(HttpClient.HEADER_X_TAXII_CONTENT_TYPE, Versions.VID_TAXII_XML_11)
                    .header(HttpClient.HEADER_X_TAXII_SERVICES, Versions.VID_TAXII_SERVICES_11)
                    .build();
    }
    
    private String getProtocol(HttpServletRequest request) throws Exception {
        String scheme = new URI(request.getRequestURL().toString()).getScheme();
        if(scheme != null && scheme.equalsIgnoreCase("https")) {
            return Versions.VID_TAXII_HTTPS_10;
        } else {
            return Versions.VID_TAXII_HTTP_10;
        }
    }
    
    private void printHeaders(HttpServletRequest request) {
        System.out.println("--------------------------------------------");
        List headerNames = Collections.list(request.getHeaderNames());         
        for(String name : headerNames) {
            System.out.println(name + ": " + request.getHeader(name));
        }
    }

    private String toXml(Object discoveryResponse) throws Exception {
        final Marshaller m = taxiiXml.createMarshaller(false); 
        m.setProperty(Marshaller.JAXB_FRAGMENT, true); // Don't generate xml declaration.
        final StringWriter sw = new StringWriter();
        m.marshal(discoveryResponse, sw);
        return sw.toString();
    }

    private Object getRequestObject(String x) throws Exception {
        Unmarshaller um = taxiiXml.getJaxbContext().createUnmarshaller();
        return um.unmarshal(new StringReader(x));
    }

    private XMLGregorianCalendar getTimestamp(Date dte) throws Exception {
        if(dte == null) {
            dte = new Date();
        }
        GregorianCalendar gc = new GregorianCalendar();
        gc.setTime(dte);
        return DatatypeFactory.newInstance().newXMLGregorianCalendar(gc);
    }