implemented claims index and recent claims

sql/dbg.sql

@@ -0,0 +1,52 @@
+DELIMITER //
+
+CREATE PROCEDURE DeleteBlock (
+    IN BlockId BIGINT
+)
+BEGIN
+    START TRANSACTION;
+
+    DELETE FROM InputsAddresses WHERE InputId IN (
+        SELECT Id FROM Inputs WHERE TransactionId IN (
+            SELECT Id FROM Transactions WHERE BlockHash IN (
+                SELECT Hash FROM Blocks WHERE Id = BlockId
+            )
+        )
+    );
+
+    DELETE FROM OutputsAddresses WHERE OutputId IN (
+        SELECT Id FROM Outputs WHERE TransactionId IN (
+            SELECT Id FROM Transactions WHERE BlockHash IN (
+                SELECT Hash FROM Blocks WHERE Id = BlockId
+            )
+        )
+    );
+
+    DELETE FROM Inputs WHERE TransactionId IN (
+        SELECT Id FROM Transactions WHERE BlockHash IN (
+            SELECT Hash FROM Blocks WHERE Id = BlockId
+        )
+    );
+
+    DELETE FROM Outputs WHERE TransactionId IN (
+        SELECT Id FROM Transactions WHERE BlockHash IN (
+            SELECT Hash FROM Blocks WHERE Id = BlockId
+        )
+    );
+
+    DELETE FROM TransactionsAddresses WHERE TransactionId IN (
+        SELECT Id FROM Transactions WHERE BlockHash IN (
+            SELECT Hash FROM Blocks WHERE Id = BlockId
+        )
+    );
+
+    DELETE FROM Transactions WHERE BlockHash IN (
+        SELECT Hash FROM Blocks WHERE Id = BlockId
+    );
+
+    DELETE FROM Blocks WHERE Id = BlockId;
+
+    COMMIT;
+END//
+
+DELIMITER ;

sql/lbryexplorer.ddl.sql

@@ -174,24 +174,46 @@ CREATE TABLE `Claims`
 (
     `Id` SERIAL,
     `TransactionHash` VARCHAR(70) CHARACTER SET latin1 COLLATE latin1_general_ci,
-    `Name` VARCHAR(200) NOT NULL,
+    `Vout` INTEGER UNSIGNED NOT NULL,
+    `Name` VARCHAR(1024) NOT NULL,
     `ClaimId` CHAR(40) CHARACTER SET latin1 COLLATE latin1_general_ci NOT NULL,
     `ClaimType` TINYINT(1) NOT NULL, -- 1 - CertificateType, 2 - StreamType
     `PublisherId` CHAR(40) CHARACTER SET latin1 COLLATE latin1_general_ci COMMENT 'references a ClaimId with CertificateType',
     `PublisherSig` VARCHAR(200) CHARACTER SET latin1 COLLATE latin1_general_ci,
     `Certificate` TEXT,
-    `Stream` TEXT,
     `TransactionTime` INTEGER UNSIGNED,
     `Version` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_general_ci NOT NULL,
+
+    -- Additional fields for easy indexing of stream types
+    `Author` VARCHAR(512),
+    `Description` MEDIUMTEXT,
+    `ContentType` VARCHAR(162) CHARACTER SET latin1 COLLATE latin1_general_ci,
+    `IsNSFW` TINYINT(1) DEFAULT 0 NOT NULL,
+    `Language` VARCHAR(20) CHARACTER SET latin1 COLLATE latin1_general_ci,
+    `ThumbnailUrl` TEXT,
+    `Title` TEXT,
+
     `Created` DATETIME NOT NULL,
     `Modified` DATETIME NOT NULL,
     PRIMARY KEY `PK_Claim` (`Id`),
     FOREIGN KEY `FK_ClaimTransaction` (`TransactionHash`) REFERENCES `Transactions` (`Hash`),
     FOREIGN KEY `FK_ClaimPublisher` (`PublisherId`) REFERENCES `Claims` (`ClaimId`),
-    CHECK((`ClaimType` = 1 AND JSON_VALID(`Certificate`)) -- certificate type
+    CONSTRAINT `Cnt_ClaimCertificate` CHECK(`Certificate` IS NULL OR JSON_VALID(`Certificate`)), -- certificate type
-          OR (`ClaimType` = 2 AND JSON_VALID(`Stream`))), -- stream type
     INDEX `Idx_Claim` (`ClaimId`),
     INDEX `Idx_ClaimTransactionTime` (`TransactionTime`),
     INDEX `Idx_ClaimCreated` (`Created`),
-    INDEX `Idx_ClaimModified` (`Modified`)
+    INDEX `Idx_ClaimModified` (`Modified`),
+
+    INDEX `Idx_ClaimAuthor` (`Author`(191)),
+    INDEX `Idx_ClaimContentType` (`ContentType`),
+    INDEX `Idx_ClaimLanguage` (`Language`),
+    INDEX `Idx_ClaimTitle` (`Title`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4;
+
+CREATE TABLE ClaimStreams
+(
+    `Id` BIGINT UNSIGNED NOT NULL,
+    `Stream` MEDIUMTEXT NOT NULL,
+    PRIMARY KEY `PK_ClaimStream` (`Id`),
+    FOREIGN KEY `PK_ClaimStreamClaim` (`Id`) REFERENCES `Claims` (`Id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4;

src/Controller/MainController.php

@@ -90,6 +90,7 @@ class MainController extends AppController {
 
     public function index() {
         $this->loadModel('Blocks');
+        $this->loadModel('Claims');
 
         $lbcUsdPrice = $this->_getLatestPrice();
         $this->set('lbcUsdPrice', $lbcUsdPrice);
@@ -101,11 +102,15 @@ class MainController extends AppController {
             $blocks[$i]->TransactionCount = count($tx_hashes);
         }
 
-        // try to calculate the hashrate based on the last 12 blocks found
+        // hash rate
-        $diffBlocks = $this->Blocks->find()->select(['Chainwork', 'BlockTime', 'Difficulty'])->order(['Height' => 'desc'])->limit(12)->toArray();
+        $hashRate = $this->_formatHashRate($this->_gethashrate());
-        $hashRate = $this->_formatHashRate($this->_gethashrate()) . '/s';
+
+        // recent claims
+        $claims = $this->Claims->find()->select(['TransactionHash', 'Name', 'Vout', 'ClaimId', 'ClaimType', 'Author', 'Title', 'Description', 'ContentType',
+                                                 'IsNSFW', 'Language', 'ThumbnailUrl', 'Created'])->contain(['Publisher' => ['fields' => ['Name']]])->order(['Claims.Created' => 'DESC'])->limit(5)->toArray();
 
         $this->set('recentBlocks', $blocks);
+        $this->set('recentClaims', $claims);
         $this->set('hashRate', $hashRate);
     }
 
@@ -155,20 +160,24 @@ class MainController extends AppController {
     }
 
     protected function _formatHashRate($value) {
+        if ($value === 'N/A') {
+            return $value;
+        }
+
         /*if ($value > 1000000000000) {
             return number_format( $value / 1000000000000, 2, '.', '' ) . ' TH';
         }*/
         if ($value > 1000000000) {
-            return number_format( $value / 1000000000, 2, '.', '' ) . ' GH';
+            return number_format( $value / 1000000000, 2, '.', '' ) . ' GH/s';
         }
         if ($value > 1000000) {
-            return number_format( $value / 1000000, 2, '.', '' ) . ' MH';
+            return number_format( $value / 1000000, 2, '.', '' ) . ' MH/s';
         }
         if ($value > 1000) {
-            return number_format( $value / 1000, 2, '.', '' ) . ' KH';
+            return number_format( $value / 1000, 2, '.', '' ) . ' KH/s';
         }
 
-        return number_format($value) . ' H';
+        return number_format($value) . ' H/s';
     }
 
     public function find() {
@@ -420,11 +429,15 @@ class MainController extends AppController {
 
     private function _gethashrate() {
         $req = ['method' => 'getnetworkhashps', 'params' => []];
+        try {
             $res = json_decode(self::curl_json_post(self::rpcurl, json_encode($req)));
             if (!isset($res->result)) {
                 return 0;
             }
             return $res->result;
+        } catch (\Exception $e) {
+            return 'N/A';
+        }
     }
 
     public function apistatus() {
@@ -440,8 +453,7 @@ class MainController extends AppController {
         $lbcUsdPrice = $this->_getLatestPrice();
 
         // Calculate hash rate
-        $diffBlocks = $this->Blocks->find()->select(['Chainwork', 'BlockTime', 'Difficulty'])->order(['Height' => 'desc'])->limit(12)->toArray();
+        $hashRate = $this->_formatHashRate($this->_gethashrate());
-        $hashRate = $this->_formatHashRate($this->_gethashrate()) . '/s';
 
         return $this->_jsonResponse(['success' => true, 'status' => [
             'height' => $height,

src/Model/Entity/Claim.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Model\Entity;
+
+use Cake\ORM\Entity;
+
+class Claim extends Entity {
+
+}
+
+?>

src/Model/Entity/ClaimStream.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Model\Entity;
+
+use Cake\ORM\Entity;
+
+class ClaimStream extends Entity {
+
+}
+
+?>

src/Model/Table/ClaimStreamsTable.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Model\Table;
+
+use Cake\ORM\Table;
+
+class ClaimStreamsTable extends Table {
+    public function initialize(array $config) {
+        parent::initialize($config);
+
+        $this->primaryKey('Id');
+        $this->table('ClaimStreams');
+
+        //$this->addBehavior('SimpleAudit');
+    }
+}
+
+?>

src/Model/Table/ClaimsTable.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Model\Table;
+
+use Cake\ORM\Table;
+
+class ClaimsTable extends Table {
+    public function initialize(array $config) {
+        parent::initialize($config);
+
+        $this->primaryKey('Id');
+        $this->table('Claims');
+
+        //$this->addBehavior('SimpleAudit');
+        $this->addAssociations([
+            'belongsTo' => [
+                'Publisher' => [
+                    'className' => 'App\Model\Table\ClaimsTable',
+                    'foreignKey' => 'PublisherId',
+                    'bindingKey' => 'ClaimId',
+                    'propertyName' => 'Publisher'
+                ]
+            ]
+        ]);
+    }
+}
+
+?>

src/Shell/BlockShell.php

@@ -24,6 +24,8 @@ class BlockShell extends Shell {
         parent::initialize();
         $this->loadModel('Blocks');
         $this->loadModel('Addresses');
+        $this->loadModel('Claims');
+        $this->loadModel('ClaimStreams');
         $this->loadModel('Inputs');
         $this->loadModel('Outputs');
         $this->loadModel('Transactions');
@@ -38,6 +40,249 @@ class BlockShell extends Shell {
         $decoded = self::decode_tx($raw);
     }
 
+    public static function hex2str($hex){
+        $string = '';
+        for ($i = 0; $i < strlen($hex)-1; $i+=2){
+            $string .= chr(hexdec($hex[$i].$hex[$i+1]));
+        }
+        return $string;
+    }
+
+    public function buildclaimindex() {
+        self::lock('buildindex');
+
+        // start with all txs
+        $decoder_url = 'http://127.0.0.1:5000';
+        $redis = self::_init_redis();
+        $conn = ConnectionManager::get('default');
+        $redis_key = 'claim.oid';
+        $last_claim_oid = $redis->exists($redis_key) ? $redis->get($redis_key) : 0;
+        try {
+            $stmt = $conn->execute('SELECT COUNT(Id) AS RecordCount FROM Outputs WHERE Id > ?', [$last_claim_oid]);
+            $count = min(500000, $stmt->fetch(\PDO::FETCH_OBJ)->RecordCount);
+
+            $idx = 0;
+            $stmt = $conn->execute('SELECT O.Id, O.TransactionId, O.Vout, O.ScriptPubKeyAsm, T.Hash, IFNULL(T.TransactionTime, T.CreatedTime) AS TxTime FROM Outputs O JOIN Transactions T ON T.Id = O.TransactionId WHERE O.Id > ? ORDER BY O.Id ASC LIMIT 500000', [$last_claim_oid]);
+            while ($out = $stmt->fetch(\PDO::FETCH_OBJ)) {
+                $idx++;
+                $idx_str = str_pad($idx, strlen($count), '0', STR_PAD_LEFT);
+
+                $txid = $out->TransactionId;
+                $vout = $out->Vout;
+
+                if (strpos($out->ScriptPubKeyAsm, 'OP_CLAIM_NAME') !== false) {
+                    $asm_parts = explode(' ', $out->ScriptPubKeyAsm, 4);
+                    $name_hex = $asm_parts[1];
+                    $claim_name = @pack('H*', $name_hex);
+
+                    // decode claim
+                    $url = sprintf("%s/claim_decode/%s", $decoder_url, $claim_name);
+                    $json = null;
+                    try {
+                        $json = self::curl_json_get($url);
+                    } catch (\Exception $e) {
+                        echo "[$idx_str/$count] claimdecode failed for [$out->Hash:$vout]. Skipping.\n";
+                        continue;
+                    }
+                    $claim = json_decode($json);
+
+                    if ($claim) {
+                        $req = ['method' => 'getvalueforname', 'params' => [$claim_name]];
+                        $json = null;
+                        try {
+                            $json = json_decode(self::curl_json_post(self::rpcurl, json_encode($req)));
+                            if (!$json) {
+                                echo "[$idx_str/$count] getvalueforname failed for [$out->Hash:$vout]. Skipping.\n";
+                                continue;
+                            }
+                        } catch (\Exception $e) {
+                            echo "[$idx_str/$count] getvalueforname failed for [$out->Hash:$vout]. Skipping.\n";
+                            continue;
+                        }
+
+                        echo "[$idx_str/$count] claim found for [$out->Hash:$vout]. Processing claim... \n";
+                        $claim_data = [];
+
+                        $claim_id = $json->result->claimId;
+                        $tx_dt = \DateTime::createFromFormat('U', $out->TxTime);
+
+                        $claim_stream_data = null;
+                        if ($claim->claimType === 'streamType') {
+                            // Build claim object to save
+                            $claim_data = [
+                                'ClaimId' => $claim_id,
+                                'TransactionHash' => $out->Hash,
+                                'Vout' => $out->Vout,
+                                'Name' => $claim_name,
+                                'Version' => $claim->version,
+                                'ClaimType' => 2, // streamType
+                                'ContentType' => isset($claim->stream->source->contentType) ? $claim->stream->source->contentType : null,
+                                'Title' => isset($claim->stream->metadata->title) ? $claim->stream->metadata->title : null,
+                                'Description' => isset($claim->stream->metadata->description) ? $claim->stream->metadata->description : null,
+                                'Language' => isset($claim->stream->metadata->language) ? $claim->stream->metadata->language : null,
+                                'Author' => isset($claim->stream->metadata->author) ? $claim->stream->metadata->author : null,
+                                'ThumbnailUrl' => isset($claim->stream->metadata->thumbnail) ? $claim->stream->metadata->thumbnail : null,
+                                'IsNSFW' => isset($claim->stream->metadata->nsfw) ? $claim->stream->metadata->nsfw : 0,
+                                'Created' => $tx_dt->format('Y-m-d H:i:s'),
+                                'Modified' => $tx_dt->format('Y-m-d H:i:s')
+                            ];
+
+                            $claim_stream_data = [
+                                'Stream' => json_encode($claim->stream)
+                            ];
+
+                            if (isset($claim->publisherSignature)) {
+                                $sig_claim = $this->Claims->find()->select(['Id', 'ClaimId', 'Name'])->where(['ClaimId' => $claim->publisherSignature->certificateId])->first();
+                                if ($sig_claim) {
+                                    $claim_data['PublisherId'] = $sig_claim->ClaimId;
+                                    $claim_data['PublisherName'] = $sig_claim->Name;
+                                }
+                            }
+                        } else {
+                            $claim_data = [
+                                'ClaimId' => $claim_id,
+                                'TransactionHash' => $out->Hash,
+                                'Vout' => $out->Vout,
+                                'Name' => $claim_name,
+                                'Version' => $claim->version,
+                                'ClaimType' => 1,
+                                'Certificate' => json_encode($claim->certificate),
+                                'Created' => $tx_dt->format('Y-m-d H:i:s'),
+                                'Modified' => $tx_dt->format('Y-m-d H:i:s')
+                            ];
+                        }
+
+                        $conn->begin();
+                        $data_error = false;
+
+                        $claim_entity = $this->Claims->newEntity($claim_data);
+                        $res = $this->Claims->save($claim_entity);
+
+                        if (!$res) {
+                            $data_error = true;
+                            echo "[$idx_str/$count] claim for [$out->Hash:$vout] FAILED to save.\n";
+                        }
+
+                        if (!$data_error) {
+                            if ($claim_stream_data) {
+                                $claim_stream_data['Id'] = $claim_entity->Id;
+                                $claim_stream_entity = $this->ClaimStreams->newEntity($claim_stream_data);
+
+                                $res = $this->ClaimStreams->save($claim_stream_entity);
+                                if (!$res) {
+                                    $data_error = true;
+                                }
+                            }
+                        }
+
+                        if (!$data_error) {
+                            $conn->commit();
+                            echo "[$idx_str/$count] claim for [$out->Hash:$vout] indexed.\n";
+                        } else {
+                            $conn->rollback();
+                            echo "[$idx_str/$count] claim for [$out->Hash:$vout] NOT indexed. Rolled back.\n";
+                        }
+                    } else {
+                        echo "[$idx_str/$count] claim for [$out->Hash:$vout] could not be decoded. Skipping.\n";
+                    }
+                } else {
+                    echo "[$idx_str/$count] no claim found for [$out->Hash:$vout]. Skipping.\n";
+                }
+
+                $redis->set($redis_key, $out->Id);
+            }
+        } catch (\Exception $e) {
+            // continue
+            print_r($e);
+        }
+
+        self::unlock('buildindex');
+    }
+
+    protected function _getclaimfortxout($pubkeyasm, $tx_hash, $vout, $tx_time = null) {
+        $claim_data = null;
+        $claim_stream_data = null;
+
+        $asm_parts = explode(' ', $pubkeyasm, 4);
+        $name_hex = $asm_parts[1];
+        $claim_name = @pack('H*', $name_hex);
+
+        // decode claim
+        $decoder_url = 'http://127.0.0.1:5000';
+        $url = sprintf("%s/claim_decode/%s", $decoder_url, $claim_name);
+        $json = null;
+        try {
+            $json = self::curl_json_get($url);
+        } catch (\Exception $e) {
+            echo "***claimdecode failed for [$tx_hash:$vout]. Skipping.\n";
+        }
+
+        if ($json) {
+            $claim = json_decode($json);
+            if ($claim) {
+                $req = ['method' => 'getvalueforname', 'params' => [$claim_name]];
+                $json = null;
+                try {
+                    $json = json_decode(self::curl_json_post(self::rpcurl, json_encode($req)));
+                    if ($json) {
+                        $claim_data = [];
+                        $claim_id = $json->result->claimId;
+                        $now = new \DateTime('now', new \DateTimeZone('UTC'));
+                        $tx_dt = ($tx_time != null) ? $tx_time : $now;
+                        if ($claim->claimType === 'streamType') {
+                            // Build claim object to save
+                            $claim_data = [
+                                'ClaimId' => $claim_id,
+                                'TransactionHash' => $tx_hash,
+                                'Vout' => $vout,
+                                'Name' => $claim_name,
+                                'Version' => $claim->version,
+                                'ClaimType' => 2, // streamType
+                                'ContentType' => isset($claim->stream->source->contentType) ? $claim->stream->source->contentType : null,
+                                'Title' => isset($claim->stream->metadata->title) ? $claim->stream->metadata->title : null,
+                                'Description' => isset($claim->stream->metadata->description) ? $claim->stream->metadata->description : null,
+                                'Language' => isset($claim->stream->metadata->language) ? $claim->stream->metadata->language : null,
+                                'Author' => isset($claim->stream->metadata->author) ? $claim->stream->metadata->author : null,
+                                'ThumbnailUrl' => isset($claim->stream->metadata->thumbnail) ? $claim->stream->metadata->thumbnail : null,
+                                'IsNSFW' => isset($claim->stream->metadata->nsfw) ? $claim->stream->metadata->nsfw : 0,
+                                'Created' => $tx_dt->format('Y-m-d H:i:s'),
+                                'Modified' => $tx_dt->format('Y-m-d H:i:s')
+                            ];
+
+                            $claim_stream_data = [
+                                'Stream' => json_encode($claim->stream)
+                            ];
+
+                            if (isset($claim->publisherSignature)) {
+                                $sig_claim = $this->Claims->find()->select(['Id', 'ClaimId', 'Name'])->where(['ClaimId' => $claim->publisherSignature->certificateId])->first();
+                                if ($sig_claim) {
+                                    $claim_data['PublisherId'] = $sig_claim->ClaimId;
+                                    $claim_data['PublisherName'] = $sig_claim->Name;
+                                }
+                            }
+                        } else {
+                            $claim_data = [
+                                'ClaimId' => $claim_id,
+                                'TransactionHash' => $tx_hash,
+                                'Vout' => $vout,
+                                'Name' => $claim_name,
+                                'Version' => $claim->version,
+                                'ClaimType' => 1,
+                                'Certificate' => json_encode($claim->certificate),
+                                'Created' => $tx_dt->format('Y-m-d H:i:s'),
+                                'Modified' => $tx_dt->format('Y-m-d H:i:s')
+                            ];
+                        }
+                    }
+                } catch (\Exception $e) {
+                    echo "***getvalueforname failed for [$out->Hash:$vout]. Skipping.\n";
+                }
+            }
+        }
+
+        return ['claim_data' => $claim_data, 'claim_stream_data' => $claim_stream_data];
+    }
+
     public function fixzerooutputs() {
         self::lock('zerooutputs');
 
@@ -422,7 +667,7 @@ class BlockShell extends Shell {
                         $addr_id_drcr[$addr_id]['debit'] = bcadd($addr_id_drcr[$addr_id]['debit'], $in['Value'], 8);
 
                         try {
-                            $conn->execute('REPLACE INTO InputsAddresses (InputId, AddressId) VALUES (?, ?)', [$in_entity->Id, $in['AddressId']]);
+                            $conn->execute('INSERT INTO InputsAddresses (InputId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE InputId = InputId', [$in_entity->Id, $in['AddressId']]);
                             $conn->execute('UPDATE Addresses SET TotalSent = TotalSent + ? WHERE Id = ?', [$in['Value'], $in['AddressId']]);
                             $conn->execute('INSERT INTO TransactionsAddresses (TransactionId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE TransactionId = TransactionId', [$numeric_tx_id, $in['AddressId']]);
                         } catch (\Exception $e) {
@@ -481,7 +726,7 @@ class BlockShell extends Shell {
                     $addr_id_drcr[$addr_id]['credit'] = bcadd($addr_id_drcr[$addr_id]['credit'], $out['Value'], 8);
 
                     try {
-                        $conn->execute('REPLACE INTO OutputsAddresses (OutputId, AddressId) VALUES (?, ?)', [$out_entity->Id, $out_addr_id]);
+                        $conn->execute('INSERT INTO OutputsAddresses (OutputId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE OutputId = OutputId', [$out_entity->Id, $out_addr_id]);
                         $conn->execute('UPDATE Addresses SET TotalReceived = TotalReceived + ? WHERE Id = ?', [$out['Value'], $out_addr_id]);
                         $conn->execute('INSERT INTO TransactionsAddresses (TransactionId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE TransactionId = TransactionId', [$numeric_tx_id, $out_addr_id]);
                     } catch (\Exception $e) {
@@ -490,6 +735,36 @@ class BlockShell extends Shell {
                         break;
                     }
                 }
+
+                // create the claim if the asm pub key starts with OP_CLAIM_NAME
+                if (strpos($out['ScriptPubKeyAsm'], 'OP_CLAIM_NAME') !== false) {
+                    $all_claim_data = $this->_getclaimfortxout($out['ScriptPubKeyAsm'], $tx_hash, $out['Vout'], $block_ts);
+                    $claim = $all_claim_data['claim_data'];
+                    $claim_stream_data = $all_claim_data['claim_stream_data'];
+                    if ($claim['ClaimType'] == 2 && !$claim_stream_data) {
+                        echo "***claim stream data missing for streamType claim\n";
+                        $data_error = true;
+                        break;
+                    }
+
+                    $claim_entity = $this->Claims->newEntity($claim);
+                    $res = $this->Claims->save($claim_entity);
+                    if (!$res) {
+                        echo "***claim could not be saved.\n";
+                        $data_error = true;
+                        break;
+                    }
+
+                    if (!$data_error && $claim_stream_data) {
+                        $claim_stream_data['Id'] = $claim_entity->Id;
+                        $claim_stream_entity = $this->ClaimStreams->newEntity($claim_stream_data);
+                        $res = $this->ClaimStreams->save($claim_stream_entity);
+                        if (!$res) {
+                            echo "***claim stream could not be saved.\n";
+                            $data_error = true;
+                        }
+                    }
+                }
             }
         }
 
@@ -956,6 +1231,28 @@ class BlockShell extends Shell {
         return $response;
     }
 
+    private static function curl_json_get($url, $headers = []) {
+        $ch = curl_init();
+        curl_setopt($ch, CURLOPT_URL, $url);
+        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+
+        $response = curl_exec($ch);
+        //Log::debug('Request execution completed.');
+        if ($response === false) {
+            $error = curl_error($ch);
+            $errno = curl_errno($ch);
+            curl_close($ch);
+
+            throw new \Exception(sprintf('The request failed: %s', $error), $errno);
+        } else {
+            curl_close($ch);
+        }
+
+        // Close any open file handle
+        return $response;
+    }
+
     private static $base58chars = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
 
     public static $op_codes = [
@@ -1315,7 +1612,7 @@ class BlockShell extends Shell {
                 }
 
                 if ($addr_id > -1) {
-                    $conn->execute('REPLACE INTO OutputsAddresses (OutputId, AddressId) VALUES (?, ?)', [$out->Id, $addr_id]);
+                    $conn->execute('INSERT INTO OutputsAddresses (OutputId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE OutputId = OutputId', [$out->Id, $addr_id]);
                 }
             }
 
@@ -1350,7 +1647,7 @@ class BlockShell extends Shell {
 
                 $in_entity = $this->Inputs->newEntity($in_data);
                 if ($this->Inputs->save($in_entity)) {
-                    $conn->execute('REPLACE INTO InputsAddresses (InputId, AddressId) VALUES (?, ?)', [$in->Id, $in_data['AddressId']]);
+                    $conn->execute('INSERT INTO InputsAddresses (InputId, AddressId) VALUES (?, ?) ON DUPLICATE KEY UPDATE InputId = InputId', [$in->Id, $in_data['AddressId']]);
                 }
             }
 
@@ -1420,7 +1717,7 @@ class BlockShell extends Shell {
             '6' => '/^OP_HASH160/',
             '7' => '/^[0-9a-f]{40}$/i', // pos 8
             '8' => '/^OP_EQUALVERIFY/',
-            '9' => '/^OP_CHECKSIG/',
+            '9' => '/^OP_CHECKSIG/'
         ];
 
         // update_claim
@@ -1438,7 +1735,7 @@ class BlockShell extends Shell {
             '7' => '/^OP_HASH160/',
             '8' => '/^[0-9a-f]{40}$/i', // pos 8
             '9' => '/^OP_EQUALVERIFY/',
-            '10' => '/^OP_CHECKSIG/',
+            '10' => '/^OP_CHECKSIG/'
         ];
 
         // Standard: pay to pubkey hash

src/Template/Main/index.ctp

@@ -76,6 +76,16 @@
     $(document).ready(function() {
         setInterval(updateStatus, updateInterval);
         setInterval(updateRecentBlocks, updateInterval);
+
+        $('.claim-box img').on('error', function() {
+            var img = $(this);
+            var parent = img.parent();
+            var text = parent.attr('data-autothumb');
+            img.remove();
+            parent.append(
+                $('<div></div>').attr({'class': 'autothumb' }).text(text)
+            );
+        });
     });
 </script>
 <?php $this->end(); ?>
@@ -142,6 +152,67 @@
                 </tbody>
             </table>
         </div>
+
+        <div class="recent-claims">
+            <h3>Recent Claims</h3>
+            <?php $idx = 0; foreach ($recentClaims as $claim):
+                $idx++; $a = ['purple', 'orange', 'blue', 'teal', 'green', 'yellow'];
+                $autoThumbText = '';
+                $link = $claim->Name;
+                if (isset($claim->Publisher->Name)) {
+                    $link = $claim->Publisher->Name . '/' . $link;
+                }
+                $link = 'lbry://' . $link;
+
+                // content type
+                $ctTag = null;
+                if (substr($claim->ContentType, 0, 5) === 'audio') {
+                    $ctTag = 'audio';
+                } else if (substr($claim->ContentType, 0, 5) === 'video') {
+                    $ctTag = 'video';
+                } else if (substr($claim->ContentType, 0, 5) === 'image') {
+                    $ctTag = 'image';
+                }
+
+                if ($claim->ClaimType == 1) { $autoThumbText = strtoupper(substr($claim->Name, 1, min( strlen($claim->Name), 3 ))); } else {
+                    $str = (strlen(trim($claim->Title)) > 0) ? $claim->Title : $claim->Name;
+                    $autoThumbText = strtoupper(substr($str, 0, min (strlen($str), 2 )));
+                }
+            ?>
+            <div class="claim-box<?php if ($idx == 5): ?> last<?php endif; ?>">
+                <div class="tags">
+                    <?php if ($ctTag): ?>
+                    <div class="content-type"><?php echo strtoupper($ctTag) ?></div>
+                    <?php endif; ?>
+                    <?php if ($claim->IsNSFW): ?>
+                    <div class="nsfw">NSFW</div>
+                    <?php endif; ?>
+                </div>
+
+                <div data-autothumb="<?php echo $autoThumbText ?>" class="thumbnail <?php echo $a[mt_rand(0, count($a) - 1)] ?>">
+                    <?php if (!$claim->IsNSFW && strlen(trim($claim->ThumbnailUrl)) > 0): ?>
+                        <img src="<?php echo $claim->ThumbnailUrl ?>" alt="" />
+                    <?php else: ?>
+                        <div class="autothumb"><?php echo $autoThumbText ?></div>
+                    <?php endif; ?>
+                </div>
+
+                <div class="metadata">
+                    <div class="title"><?php echo $claim->ClaimType == 1 ? $claim->Name : ((strlen(trim($claim->Title)) > 0) ? $claim->Title : '<em>No Title</em>') ?></div>
+                    <div class="link"><a href="<?php echo $link ?>"><?php echo $link ?></a></div>
+
+                    <div class="clear"></div>
+                    <?php if ($claim->ClaimType == 2 && strlen(trim($claim->Description)) > 0): ?>
+                    <div class="desc"><?php echo $claim->Description ?></div>
+                    <?php endif; ?>
+                </div>
+
+                <a class="tx-link" href="/tx/<?php echo $claim->TransactionHash ?>#output-<?php echo $claim->Vout ?>" target="_blank">Transaction</a>
+            </div>
+            <?php endforeach; ?>
+
+            <div class="clear"></div>
+        </div>
     </div>
 
 </div>

webroot/css/main.css

@@ -24,9 +24,33 @@ border-radius: 0 6px 8px 0 }
 .home-container-cell .ctls a { font-size: 115%; display: inline-block; font-weight: 300; position: absolute; right: 0; padding-top: 12px }
 .home-container-cell .ctls a:hover { text-decoration: none; color: #1976d2 }
 
-.home-container-cell .recent-blocks { width: 1000px; margin: 48px auto 0 auto; box-shadow: 0 6px 12px rgba(0,0,0,.175); border: 1px solid rgba(0,0,0,.15); padding: 24px 36px 36px 36px; cursor: default }
+.home-container-cell .recent-blocks { width: 1000px; margin: 48px auto 0 auto; box-shadow: 0 2px 6px rgba(0,0,0,.175); border: 1px solid rgba(0,0,0,.15); padding: 24px 24px 36px 24px; cursor: default }
 .home-container-cell .recent-blocks h3 { font-weight: normal; margin: 0 0 12px 0; font-weight: 300 }
 
+.home-container-cell .recent-claims { width: 1000px; margin: 48px auto 0 auto; box-shadow: 0 2px 6px rgba(0,0,0,.175); border: 1px solid rgba(0,0,0,.15); padding: 24px 24px 36px 24px; cursor: default  }
+.home-container-cell .recent-claims h3 { font-weight: normal; margin: 0 0 12px 0; font-weight: 300 }
+.home-container-cell .recent-claims .claim-box { width: 184px; height: 330px; margin-right: 7px; float: left; box-shadow: 0 2px 4px rgba(0,0,0,.175); border: 1px solid rgba(0,0,0,.15); cursor: default; overflow: hidden; position: relative }
+.home-container-cell .recent-claims .claim-box.last { margin-right: 0 }
+.home-container-cell .recent-claims .claim-box .tags { font-size: 65%; position: absolute; right: 0; top: 0; z-index: 505 }
+.home-container-cell .recent-claims .claim-box .thumbnail { width: 100%; height: 120px; background: #f0f0f0; display: block; position: relative; overflow: hidden;}
+.home-container-cell .recent-claims .claim-box .thumbnail img { width: 100%; position: absolute; left: 0; top: 0; border-bottom: 1px solid #eee }
+.home-container-cell .recent-claims .claim-box .thumbnail.purple { background: #ab47bc }
+.home-container-cell .recent-claims .claim-box .thumbnail.orange { background: #e91e63 }
+.home-container-cell .recent-claims .claim-box .thumbnail.blue { background: #42a5f5 }
+.home-container-cell .recent-claims .claim-box .thumbnail.teal { background: #4db6ac }
+.home-container-cell .recent-claims .claim-box .thumbnail.green { background: #66bb6a }
+.home-container-cell .recent-claims .claim-box .thumbnail.yellow { background: #fdd835 }
+.home-container-cell .recent-claims .claim-box .thumbnail .autothumb { display: block; margin: 33px auto 0 auto; text-align: center; font-size: 240%; color: #fff; line-height: 54px }
+.home-container-cell .recent-claims .claim-box .tags > div { display: inline-block; padding: 4px 12px; margin-left: 2px }
+.home-container-cell .recent-claims .claim-box .tags .nsfw { background: #e53935; text-align: center; color: #fff; position: relative; left: 1px }
+.home-container-cell .recent-claims .claim-box .tags .content-type { background: #880e4f; text-align: center; color: #fff; }
+.home-container-cell .recent-claims .claim-box .metadata { padding: 12px; font-size: 90% }
+.home-container-cell .recent-claims .claim-box .title { font-size: 120%; height: 25px; line-height: 25px; overflow: hidden; text-overflow: ellipsis }
+.home-container-cell .recent-claims .claim-box .desc { font-size: 80%; font-weight: 300; height: 100px; overflow: hidden; text-overflow: ellipsis; margin: 3px 0; line-height: 20px }
+.home-container-cell .recent-claims .claim-box .link { font-size: 80%; font-weight: 300; margin-top: 3px; overflow: hidden; text-overflow: ellipsis; line-height: 20px; height: 20px }
+.home-container-cell .recent-claims .claim-box .tx-link { font-size: 80%; font-weight: 300; color: #fff; background: #1e88e5; position: absolute; bottom: 0; width: 100%; display: block; line-height: 20px; height: 32px; padding: 6px 0; text-align: center; cursor: pointer  }
+.home-container-cell .recent-claims .claim-box .tx-link:hover { text-decoration: none; background: #1976d2 }
+
 .table { width: 100%; cursor: default; border-collapse: collapse; font-size: 90% }
 .table thead tr th { border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; padding: 12px 4px }
 .table tbody tr td { padding: 8px; border-bottom: 1px solid #eee; font-weight: 300; white-space: nowrap }