fill the entire rectangle to start, then draw on top. Try
experimenting with various transparencies of paint till you see all
the effects possible.

There are a couple of ways to do this.  The important thing to note is
that it is not possible to draw on a component and clear the alpha.  You
can draw on a BufferedImage and do that however.  In the example below,
I create a mask image that is black with an alpha hole in the middle.
Then in the paintComponent() method, I draw my subject image and then
draw the mask over the top of that image.  In the portion where I have
the alpha hole, the image underneath shows through.  You could put the
mask in a separate JPanel as long as you made sure that the JPanel was
set to transparent (setOpaque(false)).

import java.awt.*;
import java.awt.event.*;
import java.awt.image.*;
import java.io.*;
import javax.imageio.*;
import javax.swing.*;

public class test1 extends JPanel {
    final BufferedImage img;
    final BufferedImage mask;

    public test1() throws IOException {
        // read the subject image
        img = ImageIO.read(new File("kittens.jpg"));
        Dimension d = new Dimension(img.getWidth(),img.getHeight());
        setPreferredSize(d);

        // create a buffered image with alpha for the mask
        mask = new BufferedImage(d.width,d.height,
         BufferedImage.TYPE_INT_ARGB);
        Graphics2D g2d = mask.createGraphics();

        // set anti-aliasing on for a smooth edge
        g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
         RenderingHints.VALUE_ANTIALIAS_ON);

        // draw black on the whole image
        g2d.setColor(Color.BLACK);
        g2d.fillRect(0,0,d.width,d.height);

        // use AlphaComposite Clear to cut an alpha hole in the mask image
        g2d.setComposite(AlphaComposite.Clear);
        g2d.fillOval(d.width/2-100,d.height/2-100,200,200);

        g2d.dispose();
    }

    public void paintComponent(Graphics g) {
        // draw the image
        g.drawImage(img,0,0,null);
        // draw the mask over the top of the image
        g.drawImage(mask,0,0,null);
    }

    public static void main(String[] args) {
        EventQueue.invokeLater(new Runnable() {
            public void run() {
                try {
                    JFrame f = new JFrame();
                    f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);

                    test1 t = new test1();

                    f.add(t,BorderLayout.CENTER);
                    f.pack();
                    f.setVisible(true);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        });
    }
}