Mohanad Alkhodari asked . 2023-10-21

How to create an attention layer for deep learning networks?

numFilters = 128;
filterSize = 5;
dropoutFactor = 0.005;
numBlocks = 4;
layer = sequenceInputLayer(numFeatures,Normalization="zerocenter",Name="input");
lgraph = layerGraph(layer);
outputName = layer.Name;
for i = 1:numBlocks
    dilationFactor = 2^(i-1);
    
    layers = [
        convolution1dLayer(filterSize,numFilters,DilationFactor=dilationFactor,Padding="causal",Name="conv1_"+i)
        layerNormalizationLayer
        spatialDropoutLayer(dropoutFactor)
        convolution1dLayer(filterSize,numFilters,DilationFactor=dilationFactor,Padding="causal")
        layerNormalizationLayer
        reluLayer
        spatialDropoutLayer(dropoutFactor)
        additionLayer(2,Name="add_"+i)];
    % Add and connect layers.
    lgraph = addLayers(lgraph,layers);
    lgraph = connectLayers(lgraph,outputName,"conv1_"+i);
    % Skip connection.
    if i == 1
        % Include convolution in first skip connection.
        layer = convolution1dLayer(1,numFilters,Name="convSkip");
        lgraph = addLayers(lgraph,layer);
        lgraph = connectLayers(lgraph,outputName,"convSkip");
        lgraph = connectLayers(lgraph,"convSkip","add_" + i + "/in2");
    else
        lgraph = connectLayers(lgraph,outputName,"add_" + i + "/in2");
    end
    
    % Update layer output name.
    outputName = "add_" + i;
end
layers = [
    globalMaxPooling1dLayer("Name",'gapl')
    fullyConnectedLayer(numClasses,Name="fc")
    softmaxLayer
    classificationLayer('Classes',unique(Y_train),'ClassWeights',weights)];
lgraph = addLayers(lgraph,layers);
lgraph = connectLayers(lgraph,outputName,"gapl");

deep learning , attention , cnn , AI, Data Science, and Statistics